@sema-agent/core/dist/tools/fs/shell-read-establish.js

Stateless, task-oriented AI agent core

import { isAbsoluteForFamily, normalizeAbsPathLexically, pathFamilyOf } from "./safety.js";
import { boundaryIsTheSandbox, canonicalPathIsAuthoritative } from "../../core/remote-env.js";
import { resolveOperandSpellings } from "./bash-readonly-classifier.js";
import { judgedCandidatesOf, sedBackupNames } from "./shell-read-write-targets.js";
import { movedReads, withHomeViews } from "./shell-read-identity.js";
const GLOB_METACHAR_RE = /[*?[]/;
export const MAX_GLOB_DIRS = 256;
export const MAX_GLOB_MATCHES = 8192;
const WRITE_LINK_HOPS = 32;
function compileGlobSegment(pattern) {
    if (pattern.includes("\\"))
        return undefined;
    let src = "";
    for (let i = 0; i < pattern.length; i++) {
        const ch = pattern[i];
        if (ch === "*") {
            src += "[^/]*";
        }
        else if (ch === "?") {
            src += "[^/]";
        }
        else if (ch === "[") {
            const cls = parseBracketExpression(pattern, i);
            if (cls === undefined)
                src += "\\[";
            else {
                src += cls.source;
                i = cls.end;
            }
        }
        else {
            src += ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
        }
    }
    try {
        return new RegExp(`^${src}$`, "s");
    }
    catch {
        return undefined;
    }
}
function parseBracketExpression(pattern, start) {
    let i = start + 1;
    let negate = false;
    if (pattern[i] === "!" || pattern[i] === "^") {
        negate = true;
        i++;
    }
    const contentStart = i;
    let body = "";
    let anyChar = false;
    for (; i < pattern.length; i++) {
        const ch = pattern[i];
        if (ch === "]" && i > contentStart) {
            if (body.length === 0 && !anyChar)
                return undefined;
            return { source: anyChar ? "[^/]" : `[${negate ? "^" : ""}${body}]`, end: i };
        }
        if (ch === "[" && (pattern[i + 1] === ":" || pattern[i + 1] === "." || pattern[i + 1] === "=")) {
            const close = pattern.indexOf(`${pattern[i + 1]}]`, i + 2);
            if (close < 0)
                return undefined;
            anyChar = true;
            i = close + 1;
            continue;
        }
        body += ch === "-" ? "-" : ch.replace(/[\]\\^]/g, "\\$&");
    }
    return undefined;
}
function globSegmentMatches(re, pattern, name) {
    if (name.startsWith(".") && !pattern.startsWith("."))
        return false;
    return re.test(name);
}
function pathSegmentsFrom(p, from, posix = false) {
    const segs = [];
    const isSep = (c) => c === "/" || (!posix && c === "\\");
    let scan = from;
    while (scan < p.length) {
        while (scan < p.length && isSep(p[scan]))
            scan++;
        const s = scan;
        while (scan < p.length && !isSep(p[scan]))
            scan++;
        if (scan > s)
            segs.push({ text: p.slice(s, scan), start: s });
    }
    return segs;
}
function basesFor(candidate, boundary) {
    if (candidate.bases === "unmodelled")
        return "unmodelled";
    if (candidate.bases.length > 0)
        return candidate.bases;
    return [boundary.cwd ?? boundary.roots[0] ?? ""];
}
function lexicalReadsOf(candidate, boundary) {
    if (candidate.expands === "unresolvable") {
        return [{ candidate, established: "unestablishable", spellings: [candidate.spelled], cause: `"${candidate.raw}" is rewritten by the shell before the program sees it` }];
    }
    const bases = basesFor(candidate, boundary);
    if (bases === "unmodelled") {
        const floors = candidate.floorBases ?? [boundary.cwd ?? boundary.roots[0] ?? ""];
        const spellings = [candidate.spelled];
        for (const floorBase of floors) {
            const viaFloor = floorBase === "" ? undefined : resolveOperandSpellings(floorBase, candidate.spelled, boundary.homeDir);
            if (viaFloor === undefined)
                continue;
            spellings.push(viaFloor.folded);
            if (viaFloor.spelled !== undefined)
                spellings.push(viaFloor.spelled);
        }
        return [{ candidate, established: "unestablishable", spellings, cause: `"${candidate.raw}" is relative to a working directory this command's text does not settle` }];
    }
    const out = [];
    const seen = new Set();
    for (const base of bases) {
        const sp = resolveOperandSpellings(base === "" ? undefined : base, candidate.spelled, boundary.homeDir);
        if (sp === undefined) {
            const key = `?${candidate.spelled}`;
            if (!seen.has(key)) {
                seen.add(key);
                out.push({ candidate, established: "unestablishable", spellings: [candidate.spelled], cause: `"${candidate.raw}" cannot be placed against ${base === "" ? "no working directory" : `"${base}"`}` });
            }
            continue;
        }
        if (seen.has(sp.folded))
            continue;
        seen.add(sp.folded);
        out.push({ candidate, established: "lexical", target: sp.folded, spellings: sp.spelled !== undefined ? [sp.folded, sp.spelled] : [sp.folded] });
    }
    return out;
}
function unfollowedBackupReads(j, boundary, cause) {
    const spellings = sedBackupNames(j.followed.operand.spelled, j.followed.suffix).flatMap((name) => lexicalReadsOf({ ...j.candidate, spelled: name }, boundary).flatMap((r) => [...(r.target !== undefined ? [r.target] : []), ...r.spellings]));
    return [{ candidate: j.candidate, established: "unestablishable", spellings: [...new Set(spellings)], cause }];
}
function writtenAt(read, j, seat) {
    if (read.candidate.face !== "write" || !(j.written ?? j.seats ?? ["landing"]).includes(seat))
        return read;
    return { ...read, written: { seat, ...(j.child === true ? { child: true } : {}) } };
}
export function establishShellReadsLexically(plan, boundary) {
    const reads = new Map();
    const nested = new Map();
    for (const segment of plan.segments) {
        reads.set(segment, judgedCandidatesOf(segment).flatMap((j) => {
            if (j.followed !== undefined)
                return unfollowedBackupReads({ ...j, followed: j.followed }, boundary, `"${j.followed.operand.raw}" names its backup through a link chain only the filesystem can follow`);
            const seat = (j.written ?? j.seats ?? ["landing"]).includes("landing") ? "landing" : "entry";
            return lexicalReadsOf(j.candidate, boundary).map((r) => writtenAt(r, j, seat));
        }));
        for (const n of segment.nested)
            nested.set(n, establishShellReadsLexically(n, boundary));
    }
    return { plan, reads, nested, roots: boundary.roots };
}
export function createShellReadEstablisher(env, readRoots, opts) {
    const lexicalOnly = boundaryIsTheSandbox(env);
    const resolverIsAuthoritative = canonicalPathIsAuthoritative(env);
    const family = pathFamilyOf({ root: readRoots[0] ?? "/" });
    const SEP = family === "posix" ? /\// : /[\\/]/;
    const TRAILING_SEP = family === "posix" ? /\/+$/ : /[\\/]+$/;
    const joinUnder = (dir, name) => `${dir.replace(TRAILING_SEP, "")}/${name}`;
    let rootsOnce = opts?.rootsCanonical === true ? Promise.resolve(readRoots) : undefined;
    const canonicalRoots = async (signal) => {
        rootsOnce ??= (async () => {
            const out = [];
            for (const r of readRoots) {
                const c = await env.canonicalPath(r, signal);
                out.push(c.ok ? c.value : r);
            }
            return out;
        })();
        return await rootsOnce;
    };
    const resolveWithoutParentSteps = async (p, signal, writeTarget = false, linkHops = WRITE_LINK_HOPS) => {
        const parts = p.split(SEP);
        const tail = [];
        let allAbsent = true;
        while (parts.length > 0) {
            const at = parts.join("/") || "/";
            const canon = await env.canonicalPath(at, signal);
            if (!canon.ok && canon.error.code === "not_found" && writeTarget && allAbsent) {
                const entry = await env.fileInfo(at, signal);
                const link = entry.ok && entry.value.kind === "symlink" && linkHops > 0 ? await env.readLink?.(at, signal) : undefined;
                if (link?.ok === true) {
                    const landing = isAbsoluteForFamily(family, link.value) ? link.value : joinUnder(parts.slice(0, -1).join("/") || "/", link.value);
                    return await resolve([landing, ...[...tail].reverse()].join("/"), signal, true, linkHops - 1);
                }
                if (entry.ok || entry.error.code !== "not_found")
                    allAbsent = false;
            }
            if (canon.ok) {
                if (tail.length === 0)
                    return { value: canon.value, exact: true };
                return { value: [canon.value.replace(TRAILING_SEP, ""), ...[...tail].reverse()].join("/"), exact: false, missing: allAbsent };
            }
            if (canon.error.code !== "not_found" && canon.error.code !== "not_directory")
                allAbsent = false;
            tail.push(parts.pop());
        }
        return undefined;
    };
    const parentOf = (canonical) => normalizeAbsPathLexically(`${canonical}/..`);
    const resolve = async (p, signal, writeTarget = false, linkHops = WRITE_LINK_HOPS) => {
        const parts = p.split(SEP);
        const first = parts.findIndex((seg) => seg === "..");
        if (first < 0)
            return await resolveWithoutParentSteps(p, signal, writeTarget, linkHops);
        const headEnd = parts.slice(0, first).reduce((n, seg) => n + seg.length + 1, 0);
        let head = await resolveWithoutParentSteps(p.slice(0, headEnd) || "/", signal, writeTarget, linkHops);
        if (head === undefined)
            return undefined;
        let pending = [];
        const flush = async () => {
            if (pending.length === 0)
                return true;
            const next = await resolveWithoutParentSteps([head.value.replace(TRAILING_SEP, ""), ...pending].join("/"), signal, writeTarget, linkHops);
            pending = [];
            if (next === undefined)
                return false;
            head = next;
            return true;
        };
        for (let i = first; i < parts.length; i++) {
            const seg = parts[i];
            if (seg === "" || seg === ".")
                continue;
            if (seg !== "..") {
                pending.push(seg);
                continue;
            }
            if (!(await flush()))
                return undefined;
            if (!head.exact)
                return head;
            head = { value: parentOf(head.value), exact: true };
        }
        return (await flush()) ? head : undefined;
    };
    const literalRootHead = (p, roots) => {
        let best;
        for (const r of [...readRoots, ...roots]) {
            if (!r)
                continue;
            const withSep = r.endsWith("/") || r.endsWith("\\") ? r : `${r}/`;
            if (p !== r && !p.startsWith(withSep))
                continue;
            if (best === undefined || r.length > best.length)
                best = r;
        }
        return best;
    };
    const expandGlobCandidate = async (p, roots, signal) => {
        const head = literalRootHead(p, roots);
        let segs = pathSegmentsFrom(p, head?.length ?? 0, family === "posix");
        let first = segs.findIndex((s) => GLOB_METACHAR_RE.test(s.text));
        if (first < 0 && head !== undefined && GLOB_METACHAR_RE.test(head)) {
            segs = pathSegmentsFrom(p, 0, family === "posix");
            first = segs.findIndex((s) => GLOB_METACHAR_RE.test(s.text));
        }
        if (first < 0)
            return undefined;
        let frontier = [p.slice(0, segs[first].start).replace(TRAILING_SEP, "") || "/"];
        let dirsUsed = 0;
        let matches = 0;
        const canonicalOfMatch = new Map();
        for (let j = first; j < segs.length; j++) {
            const seg = segs[j].text;
            if (!GLOB_METACHAR_RE.test(seg)) {
                frontier = frontier.map((d) => joinUnder(d, seg));
                continue;
            }
            const re = compileGlobSegment(seg);
            if (re === undefined)
                return { unverifiable: `the pattern "${seg}" cannot be expanded to verify what it would read` };
            const isLast = j === segs.length - 1;
            const next = [];
            for (const dir of frontier) {
                if (++dirsUsed > MAX_GLOB_DIRS)
                    return { unverifiable: `"${p}" expands over more than ${MAX_GLOB_DIRS} directories, too many to enumerate to verify` };
                const canonDir = await resolve(dir, signal);
                if (canonDir !== undefined && canonDir.missing === true)
                    continue;
                if (canonDir !== undefined && !canonDir.exact)
                    return { unverifiable: `"${dir}" cannot be resolved to verify what "${seg}" would expand to` };
                const listed = await env.listDir(canonDir?.value ?? dir, signal);
                if (!listed.ok) {
                    if (listed.error.code === "not_found" || listed.error.code === "not_directory")
                        continue;
                    return { unverifiable: `"${dir}" cannot be enumerated to verify what "${seg}" would expand to (${listed.error.code})` };
                }
                if (seg.startsWith(".")) {
                    for (const dot of [".", ".."])
                        if (re.test(dot))
                            next.push(joinUnder(dir, dot));
                }
                const canonDirHere = isLast ? canonDir?.value : undefined;
                for (const entry of listed.value) {
                    if (!globSegmentMatches(re, seg, entry.name))
                        continue;
                    if (++matches > MAX_GLOB_MATCHES)
                        return { unverifiable: `"${p}" expands to more than ${MAX_GLOB_MATCHES} names, too many to enumerate to verify` };
                    const candidate = joinUnder(dir, entry.name);
                    if (canonDirHere !== undefined && entry.kind !== "symlink")
                        canonicalOfMatch.set(candidate, joinUnder(canonDirHere, entry.name));
                    next.push(candidate);
                }
            }
            frontier = next;
            if (frontier.length === 0)
                return { paths: [], canonical: canonicalOfMatch };
        }
        return { paths: frontier, canonical: canonicalOfMatch };
    };
    const enumerateTraversal = async (rootCanonical, signal) => {
        const found = [];
        const listedDirs = new Set();
        let root;
        const stack = [{ dir: rootCanonical, spell: rootCanonical }];
        const visited = new Set([rootCanonical]);
        let dirsUsed = 0;
        while (stack.length > 0) {
            const { dir, spell } = stack.pop();
            if (++dirsUsed > MAX_GLOB_DIRS)
                return { unverifiable: `"${rootCanonical}" holds more than ${MAX_GLOB_DIRS} directories, too many to enumerate to verify what a recursive read would open` };
            const listed = await env.listDir(dir, signal);
            if (!listed.ok) {
                if (dir === rootCanonical && listed.error.code === "not_directory")
                    root = "file";
                if (listed.error.code === "not_directory" || listed.error.code === "not_found")
                    continue;
                return { unverifiable: `"${dir}" cannot be enumerated to verify what a recursive read would open (${listed.error.code})` };
            }
            listedDirs.add(dir);
            if (dir === rootCanonical)
                root = "directory";
            for (const entry of listed.value) {
                if (found.length >= MAX_GLOB_MATCHES)
                    return { unverifiable: `"${rootCanonical}" holds more than ${MAX_GLOB_MATCHES} entries, too many to enumerate to verify what a recursive read would open` };
                const path = joinUnder(spell, entry.name);
                const canonicalPath = joinUnder(dir, entry.name);
                if (entry.kind === "symlink") {
                    const canon = await env.canonicalPath(canonicalPath, signal);
                    found.push({ path, ...(canon.ok ? { canonical: canon.value } : {}), symlink: true });
                    if (canon.ok && !visited.has(canon.value)) {
                        visited.add(canon.value);
                        stack.push({ dir: canon.value, spell: path });
                    }
                    continue;
                }
                if (entry.kind === "directory") {
                    if (!visited.has(canonicalPath)) {
                        visited.add(canonicalPath);
                        stack.push({ dir: canonicalPath, spell: path });
                    }
                    continue;
                }
                found.push({ path, canonical: canonicalPath, symlink: false });
            }
        }
        const members = found.map(({ path, canonical, symlink }) => ({ path, ...(canonical !== undefined ? { canonical } : {}), node: (symlink && canonical !== undefined && listedDirs.has(canonical) ? "directory" : "file") }));
        return { ...(root !== undefined ? { root } : {}), members };
    };
    const entryOf = async (spelling, signal) => {
        const parts = spelling.split(SEP);
        const leaf = parts.pop() ?? "";
        if (leaf === "" || leaf === "." || leaf === ".." || parts.length === 0)
            return await resolve(spelling, signal, true);
        const parent = await resolve(parts.join("/") || "/", signal, true);
        if (parent === undefined)
            return undefined;
        return { value: joinUnder(parent.value, leaf), exact: parent.exact, ...(parent.missing !== undefined ? { missing: parent.missing } : {}) };
    };
    const canonicalReadOf = async (candidate, spelling, spellings, established, origin, canonicalKnown, signal, seat = "landing") => {
        const real = canonicalKnown !== undefined ? { value: canonicalKnown, exact: true } : seat === "entry" ? await entryOf(spelling, signal) : await resolve(spelling, signal, candidate.face === "write");
        if (real === undefined) {
            if (resolverIsAuthoritative)
                return { candidate, established: "unestablishable", spellings, ...(origin !== undefined ? { origin } : {}), cause: `"${spelling}" cannot be resolved to verify where it reads` };
            return { candidate, established: "lexical", target: normalizeAbsPathLexically(spelling), spellings, ...(origin !== undefined ? { origin } : {}) };
        }
        if (!real.exact && real.missing !== true) {
            return { candidate, established: "unestablishable", spellings, ...(origin !== undefined ? { origin } : {}), cause: `"${spelling}" cannot be resolved to verify where it reads` };
        }
        return { candidate, established, target: real.value, spellings, ...(origin !== undefined ? { origin } : {}) };
    };
    const followedName = async (spelled, base, homeDir, signal) => {
        let name = spelled;
        for (let hop = 0; hop <= WRITE_LINK_HOPS; hop++) {
            const at = resolveOperandSpellings(base === "" ? undefined : base, name, homeDir);
            if (at === undefined)
                return undefined;
            const walked = at.spelled ?? at.folded;
            if (TRAILING_SEP.test(walked))
                return name;
            const parts = walked.split(SEP);
            const leaf = parts.pop() ?? "";
            if (leaf === "" || leaf === "." || leaf === "..")
                return name;
            const parent = await resolve(parts.join("/") || "/", signal);
            if (parent === undefined)
                return undefined;
            if (!parent.exact)
                return parent.missing === true ? name : undefined;
            const entry = await env.fileInfo(joinUnder(parent.value, leaf), signal);
            if (!entry.ok)
                return entry.error.code === "not_found" || entry.error.code === "not_directory" ? name : undefined;
            if (entry.value.kind !== "symlink")
                return name;
            const link = await env.readLink?.(joinUnder(parent.value, leaf), signal);
            if (link?.ok !== true)
                return undefined;
            const dir = name.slice(0, name.search(/[^/]*$/));
            name = isAbsoluteForFamily(family, link.value) ? link.value : `${dir}${link.value}`;
        }
        return undefined;
    };
    const followedBackupReads = async (j, boundary, signal) => {
        const { operand, suffix } = j.followed;
        const bases = basesFor(operand, boundary);
        const unfollowed = (cause) => unfollowedBackupReads(j, boundary, cause);
        if (bases === "unmodelled")
            return unfollowed(`"${operand.raw}" is relative to a working directory this command's text does not settle`);
        const out = [];
        for (const base of bases) {
            const name = await followedName(operand.spelled, base, boundary.homeDir, signal);
            if (name === undefined) {
                out.push(...unfollowed(`"${operand.raw}" names its backup through a link chain that could not be read`));
                continue;
            }
            for (const backup of sedBackupNames(name, suffix)) {
                const at = resolveOperandSpellings(base === "" ? undefined : base, backup, boundary.homeDir);
                if (at === undefined) {
                    out.push(...unfollowed(`"${backup}" cannot be placed against "${base}"`));
                    continue;
                }
                const spellings = at.spelled !== undefined ? [at.folded, at.spelled] : [at.folded];
                for (const seat of j.seats ?? ["landing"])
                    out.push(writtenAt(await canonicalReadOf(j.candidate, at.spelled ?? at.folded, spellings, "canonical", undefined, undefined, signal, seat), j, seat));
            }
        }
        return out;
    };
    const establishSegment = async (segment, boundary, roots, signal) => {
        const out = [];
        for (const judged of judgedCandidatesOf(segment)) {
            if (judged.followed !== undefined) {
                out.push(...(await followedBackupReads({ ...judged, followed: judged.followed }, boundary, signal)));
                continue;
            }
            const candidate = judged.candidate;
            const traversal = candidate.provenance === "traversal_root";
            const resolvesItself = TRAILING_SEP.test(candidate.spelled) || [".", ".."].includes(candidate.spelled.split(SEP).pop() ?? "");
            const landsToo = (ss) => (resolvesItself && ss.length > 0 && !ss.includes("landing") ? [...ss, "landing"] : ss);
            const seats = judged.seats === undefined ? ["landing"] : landsToo(judged.seats);
            const writtenSeats = { ...judged, written: judged.written === undefined ? seats : landsToo(judged.written) };
            const atSeats = async (spelling, spellings, established, origin, canonicalKnown) => {
                const reads = [];
                for (const seat of seats) {
                    const read = writtenAt(await canonicalReadOf(candidate, spelling, spellings, established, origin, canonicalKnown, signal, seat), writtenSeats, seat);
                    const same = reads.findIndex((r) => r.established === read.established && r.target === read.target);
                    if (same < 0)
                        reads.push(read);
                    else if (reads[same].written === undefined && read.written !== undefined)
                        reads[same] = read;
                }
                return reads;
            };
            const land = async (read, spellings, folded) => {
                if (!traversal || (read.established !== "canonical" && read.established !== "expansion") || read.target === undefined) {
                    out.push(read);
                    return;
                }
                const walkedTree = await enumerateTraversal(read.target, signal);
                if ("unverifiable" in walkedTree) {
                    out.push(read);
                    out.push({ candidate, established: "unestablishable", spellings, origin: folded, cause: walkedTree.unverifiable });
                    return;
                }
                out.push({ ...read, ...(walkedTree.root !== undefined ? { node: walkedTree.root } : {}) });
                for (const m of walkedTree.members)
                    out.push({ ...(await canonicalReadOf(candidate, m.path, [m.path], "expansion", folded, m.canonical, signal)), node: m.node, walked: true });
            };
            for (const lexical of lexicalReadsOf(candidate, boundary)) {
                if (lexical.established !== "lexical") {
                    out.push(lexical);
                    continue;
                }
                const folded = lexical.target;
                const spellings = lexical.spellings;
                const walked = spellings.find((s) => s !== folded) ?? folded;
                if (candidate.expands === "glob") {
                    let matched = false;
                    for (const p of spellings) {
                        const expanded = await expandGlobCandidate(p, roots, signal);
                        if (expanded !== undefined && "unverifiable" in expanded) {
                            out.push({ candidate, established: "unestablishable", spellings, origin: folded, cause: expanded.unverifiable });
                            matched = true;
                            continue;
                        }
                        if (expanded !== undefined && expanded.paths.length > 0) {
                            matched = true;
                            for (const t of expanded.paths)
                                for (const read of await atSeats(t, [t, folded], "expansion", folded, expanded.canonical.get(t)))
                                    await land(read, spellings, folded);
                        }
                    }
                    if (!matched)
                        for (const read of await atSeats(walked, spellings, "canonical", undefined, undefined))
                            await land(read, spellings, folded);
                    continue;
                }
                for (const read of await atSeats(walked, spellings, "canonical", undefined, undefined))
                    await land(read, spellings, folded);
            }
        }
        return out;
    };
    const establishPlan = async (plan, boundary, roots, signal) => {
        const reads = new Map();
        const nested = new Map();
        for (const segment of plan.segments) {
            reads.set(segment, await establishSegment(segment, boundary, roots, signal));
            for (const n of segment.nested)
                nested.set(n, await establishPlan(n, boundary, roots, signal));
        }
        return { plan, reads, nested, roots };
    };
    return {
        lexicalOnly,
        establish: async (plan, boundary, signal) => {
            if (lexicalOnly)
                return establishShellReadsLexically(plan, boundary);
            const roots = await canonicalRoots(signal);
            return await withHomeViews(env, opts, await establishPlan(plan, { ...boundary, roots }, roots, signal), signal);
        },
        reestablish: async (previous, boundary, signal) => {
            if (lexicalOnly)
                return { changed: [] };
            const again = await establishPlan(previous.plan, { ...boundary, roots: previous.roots }, previous.roots, signal);
            return { changed: movedReads(previous, again) };
        },
    };
}