@sema-agent/core
Version:
Stateless, task-oriented AI agent core
1,488 lines • 65 kB
JavaScript
import { isTimeCutCode } from "../../internal/harness.js";
import { delimitUntrusted } from "../../core/untrusted-text.js";
import { MAX_EXEC_OUTPUT_BYTES } from "../../core/exec-output-tail.js";
import { anchorSpelling, hasBinaryExtension, isWinFormPath, pathFamilyOf } from "./safety.js";
import { HomeViewUnreadError, homeViewsOf } from "./read-deny-home.js";
import { createEntryDenyJudge } from "./dir-entries.js";
import { thrownText } from "../../core/thrown-value.js";
const WALK_MAX_FILES = 5000;
const WALK_MAX_DEPTH = 32;
const GREP_DEFAULT_CAP = 250;
const FILE_MAX_BYTES = 5 * 1024 * 1024;
export const JS_GREP_OUTPUT_MAX_BYTES = MAX_EXEC_OUTPUT_BYTES;
export const DEFAULT_IGNORE_DIRS = new Set([
"node_modules", ".git", ".hg", ".svn", ".bzr", ".jj", ".sl", "build", "dist", "out", "target",
".dart_tool", ".pub-cache", ".next", ".nuxt", ".gradle", ".idea", ".vscode", "Pods", ".venv", "venv",
"__pycache__", ".mypy_cache", ".pytest_cache", "coverage", ".turbo", ".cache", "vendor",
]);
const VCS_DIRS = [".git", ".svn", ".hg", ".bzr", ".jj", ".sl"];
const TYPE_GLOBS = {
js: "*.{js,jsx,mjs,cjs}",
ts: "*.{ts,tsx,mts,cts}",
py: "*.{py,pyi}",
rust: "*.rs",
go: "*.go",
java: "*.java",
c: "*.{c,h}",
cpp: "*.{cpp,cc,cxx,hpp,hh}",
ruby: "*.rb",
php: "*.php",
swift: "*.swift",
kotlin: "*.{kt,kts}",
csharp: "*.cs",
sh: "*.{sh,bash,zsh}",
md: "*.{md,mdx}",
markdown: "*.{md,mdx}",
json: "*.json",
yaml: "*.{yml,yaml}",
html: "*.{html,htm}",
css: "*.{css,scss,less}",
};
const basename = (p) => p.slice(Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\")) + 1);
function collapseDotSegments(p) {
const win = isWinFormPath(p);
const lead = (win ? /^[\\/]*/ : /^\/*/).exec(p)?.[0] ?? "";
const sep = win ? "\\" : "/";
const parts = p.slice(lead.length).split(win ? /[\\/]+/ : /\/+/);
const floor = !win ? 0 : lead.length >= 2 ? 2 : /^[A-Za-z]:$/.test(parts[0] ?? "") ? 1 : 0;
const kept = [];
for (const part of parts) {
if (part === "" || part === ".")
continue;
if (part === "..") {
if (kept.length > floor)
kept.pop();
continue;
}
kept.push(part);
}
return kept.length === 0 ? "" : lead + kept.join(sep);
}
const NO_MATCHES = "No matches.";
const GREP_LINE_CLIP = 500;
function clipLine(line) {
return line.length > GREP_LINE_CLIP ? `${line.slice(0, GREP_LINE_CLIP)}… [+${line.length - GREP_LINE_CLIP} chars]` : line;
}
export function splitGlobParam(glob) {
const out = [];
for (const token of glob.split(/\s+/)) {
if (token.includes("{") && token.includes("}")) {
if (token.length > 0)
out.push(token);
}
else
out.push(...token.split(",").filter(Boolean));
}
return out;
}
export function shellQuote(s) {
return `'${s.replace(/'/g, `'\\''`)}'`;
}
function globTokenToRegExp(token, anchored, glob = false, report) {
const charToRe = (ch) => ch === "*" ? "[^/]*" : ch === "?" ? "[^/]" : ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
let re = "";
for (let i = 0; i < token.length; i++) {
const c = token[i];
if (c === "*") {
if (token[i + 1] === "*") {
re += ".*";
i++;
if (token[i + 1] === "/")
i++;
}
else
re += "[^/]*";
}
else if (c === "?")
re += "[^/]";
else if (glob && c === "{") {
const close = token.indexOf("}", i);
if (close > i + 1) {
const alts = token.slice(i + 1, close).split(",").map((alt) => [...alt].map(charToRe).join(""));
re += `(?:${alts.join("|")})`;
i = close;
}
else {
re += "\\{";
if (report)
report.degraded = true;
}
}
else if (glob && c === "[") {
let j = i + 1;
let neg = false;
if (token[j] === "!" || token[j] === "^") {
neg = true;
j++;
}
const bodyStart = j;
if (token[j] === "]")
j++;
const close = token.indexOf("]", j);
let cls;
if (close !== -1) {
const body = token.slice(bodyStart, close).replace(/[\\[\]]/g, "\\$&").replace(/\//g, "");
cls = `[${neg ? "^/" : ""}${body}]`;
try {
new RegExp(cls);
}
catch {
cls = undefined;
}
}
if (cls !== undefined) {
re += cls;
i = close;
}
else {
re += "\\[";
if (report)
report.degraded = true;
}
}
else
re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&");
}
return new RegExp(`${anchored ? "^" : "(^|/)"}${re}$`);
}
export function invalidGlobTokens(glob) {
const bad = [];
for (const token of splitGlobParam(glob)) {
const report = { degraded: false };
globTokenToRegExp(normalizeGlobToken(token), globIsAnchored(token), true, report);
if (report.degraded)
bad.push(token);
}
return bad;
}
function normalizeGlobToken(token) {
let t = token;
while (t.startsWith("./"))
t = t.slice(2);
return t.replace(/^\/+/, "");
}
function globIsAnchored(token) {
let t = token;
while (t.startsWith("./"))
t = t.slice(2);
return t.includes("/");
}
export function gitignoreMatcher(content) {
const rules = content
.split(/\r?\n/)
.map((l) => l.trim())
.filter((l) => l.length > 0 && !l.startsWith("#"))
.map((raw) => {
let neg = false;
let pat = raw;
if (pat.startsWith("!")) {
neg = true;
pat = pat.slice(1);
}
let dirOnly = false;
if (pat.endsWith("/")) {
dirOnly = true;
pat = pat.slice(0, -1);
}
const anchored = pat.startsWith("/");
if (anchored)
pat = pat.replace(/^\/+/, "");
return { neg, dirOnly, anchored, re: globTokenToRegExp(pat, anchored), bare: pat };
});
return (relRaw, isDir) => {
const rel = relRaw.replace(/\\/g, "/");
let ignored = false;
for (const r of rules) {
if (r.dirOnly && !isDir)
continue;
const hit = r.re.test(rel) || (!r.anchored && (r.re.test(basename(rel)) || rel.split("/").includes(r.bare)));
if (hit)
ignored = !r.neg;
}
return ignored;
};
}
export async function buildIgnore(env, root, signal) {
let gi;
const giPath = `${root.replace(/\/+$/, "")}/.gitignore`;
const read = await env.readTextFile(giPath, signal).catch(() => undefined);
if (read && read.ok)
gi = gitignoreMatcher(read.value);
return (rel, isDir) => {
const base = basename(rel);
if (isDir && DEFAULT_IGNORE_DIRS.has(base))
return true;
if (gi && gi(rel, isDir))
return true;
return false;
};
}
export function walkIsPartial(w) {
return w.incomplete || w.skippedUnreadable > 0 || w.aborted;
}
export async function walk(env, root, start, ignore, signal, deny) {
const out = [];
const nameOnly = [];
const mtimes = new Map();
let incomplete = false;
let skippedLarge = 0;
let skippedUnreadable = 0;
let aborted = false;
let denyPruned = 0;
const denyPatterns = [];
const judge = deny !== undefined ? createEntryDenyJudge(env, deny, signal) : undefined;
const denyHit = async (path, leaf) => {
const hit = judge === undefined ? null : await judge(path, leaf);
if (!hit)
return false;
denyPruned++;
if (!denyPatterns.includes(hit.pattern))
denyPatterns.push(hit.pattern);
return true;
};
const rootPrefix = root.replace(/[\\/]+$/, "") + (root.includes("\\") ? "\\" : "/");
const rel = (abs) => (abs.startsWith(rootPrefix) ? abs.slice(rootPrefix.length) : abs);
const stack = [{ dir: start, depth: 0 }];
const visited = new Set();
let steps = 0;
while (stack.length > 0 && out.length < WALK_MAX_FILES) {
if (++steps % SCAN_YIELD_EVERY === 0)
await yieldEventLoop();
if (signal?.aborted) {
aborted = true;
break;
}
const { dir, depth } = stack.pop();
if (visited.has(dir))
continue;
visited.add(dir);
const listing = await env.listDir(dir, signal);
if (!listing.ok) {
if (signal?.aborted) {
aborted = true;
break;
}
skippedUnreadable++;
continue;
}
for (const info of listing.value) {
if (info.kind === "directory") {
if (ignore(rel(info.path), true) || visited.has(info.path))
continue;
if (await denyHit(info.path))
continue;
if (depth < WALK_MAX_DEPTH)
stack.push({ dir: info.path, depth: depth + 1 });
else
incomplete = true;
}
else if (info.kind === "file") {
if (ignore(rel(info.path), false))
continue;
if (await denyHit(info.path, "file"))
continue;
if (typeof info.mtimeMs === "number")
mtimes.set(info.path, info.mtimeMs);
if (hasBinaryExtension(info.path)) {
if (nameOnly.length < WALK_MAX_FILES)
nameOnly.push(info.path);
else
incomplete = true;
continue;
}
if (typeof info.size === "number" && info.size > FILE_MAX_BYTES) {
skippedLarge++;
if (nameOnly.length < WALK_MAX_FILES)
nameOnly.push(info.path);
else
incomplete = true;
continue;
}
out.push(info.path);
if (out.length >= WALK_MAX_FILES) {
incomplete = true;
break;
}
}
}
}
if (stack.length > 0 && out.length >= WALK_MAX_FILES)
incomplete = true;
return { files: out, nameOnly, incomplete, skippedLarge, skippedUnreadable, aborted, mtimes, denyPruned, denyPatterns };
}
export function denyWithheldNote(w) {
if (w.denyPruned <= 0)
return "";
return `\n[note: ${w.denyPruned} entr${w.denyPruned === 1 ? "y" : "ies"} matching the sensitive-path read deny list ${w.denyPruned === 1 ? "was" : "were"} withheld from this traversal (pruned entries' descendants not counted; patterns: ${w.denyPatterns.join(", ")})]`;
}
function walkCaveat(w, forContentSearch) {
const parts = [];
if (w.aborted)
parts.push("the search was interrupted before finishing");
if (w.skippedUnreadable > 0) {
parts.push(`${w.skippedUnreadable} director${w.skippedUnreadable === 1 ? "y" : "ies"} could not be read and ${w.skippedUnreadable === 1 ? "was" : "were"} skipped`);
}
if (w.incomplete)
parts.push(`file walk hit its ${WALK_MAX_FILES}-file/${WALK_MAX_DEPTH}-depth limit`);
if (forContentSearch && w.skippedLarge > 0) {
parts.push(`${w.skippedLarge} file(s) over ${Math.round(FILE_MAX_BYTES / (1024 * 1024))}MB not searched`);
}
return parts.length > 0 ? `\n…[results may be incomplete: ${parts.join("; ")}]` : "";
}
const SCAN_YIELD_EVERY = 32;
const yieldEventLoop = () => new Promise((resolve) => (typeof setImmediate === "function" ? setImmediate(resolve) : setTimeout(resolve, 0)));
const REDOS_BOUNDED_CEILING = 100;
const REDOS_AMBIGUOUS_REPEAT_CEILING = 10;
function analyzeRedos(pattern, ignoreCase = false) {
const quantAt = (i) => {
const c = pattern[i];
if (c === "*" || c === "+")
return { len: 1, unbounded: true, max: Infinity, range: true, brace: false };
if (c === "?")
return { len: 1, unbounded: false, max: 1, range: true, brace: false };
if (c === "{") {
const m = /^\{(\d+)(?:,(\d*))?\}/.exec(pattern.slice(i));
if (!m)
return undefined;
const lo = Number(m[1]);
const hi = m[2] === "" || m[2] === undefined ? (m[2] === "" ? Infinity : lo) : Number(m[2]);
return { len: m[0].length, unbounded: hi > REDOS_BOUNDED_CEILING, max: hi, range: lo < hi, brace: true };
}
return undefined;
};
let hasNamedGroup = false;
const escTokenLen = (s, i) => {
const c = s[i + 1];
if (c === "x" && /^[0-9a-fA-F]{2}$/.test(s.slice(i + 2, i + 4)))
return 4;
if (c === "u") {
if (/^[0-9a-fA-F]{4}$/.test(s.slice(i + 2, i + 6)))
return 6;
return 2;
}
if (c === "c" && /^[a-zA-Z]$/.test(s[i + 2] ?? ""))
return 3;
if (c === "k" && hasNamedGroup && s[i + 2] === "<") {
const close = s.indexOf(">", i + 3);
if (close > i + 3)
return close - i + 1;
}
if (c === "0") {
let j = i + 2;
while (j < s.length && j < i + 4 && s[j] >= "0" && s[j] <= "7")
j++;
return j - i;
}
if (c !== undefined && c >= "1" && c <= "9") {
let j = i + 2;
while (j < s.length && j < i + 10 && s[j] >= "0" && s[j] <= "9")
j++;
if (Number(s.slice(i + 1, j)) <= captureGroups)
return j - i;
if (c >= "8")
return 2;
const maxLen = c <= "3" ? 3 : 2;
let k = i + 1;
while (k < j && k - i - 1 < maxLen && s[k] >= "0" && s[k] <= "7")
k++;
return k - i;
}
return 2;
};
let captureGroups = 0;
for (let i = 0; i < pattern.length; i++) {
const ch = pattern[i];
if (ch === "\\") {
i++;
continue;
}
if (ch === "[") {
i++;
if (pattern[i] === "^")
i++;
while (i < pattern.length && pattern[i] !== "]") {
if (pattern[i] === "\\")
i++;
i++;
}
continue;
}
if (ch === "(") {
if (pattern[i + 1] !== "?")
captureGroups++;
else if (pattern[i + 2] === "<" && pattern[i + 3] !== "=" && pattern[i + 3] !== "!") {
captureGroups++;
hasNamedGroup = true;
}
}
}
const readClassAtom = (s, pos) => {
const ch = s[pos];
if (ch === undefined)
return undefined;
if (ch !== "\\")
return { cp: ch, len: 1 };
const c = s[pos + 1];
if (c === undefined)
return undefined;
if (c === "x" && /^[0-9a-fA-F]{2}$/.test(s.slice(pos + 2, pos + 4)))
return { cp: String.fromCharCode(parseInt(s.slice(pos + 2, pos + 4), 16)), len: 4 };
if (c === "u" && /^[0-9a-fA-F]{4}$/.test(s.slice(pos + 2, pos + 6)))
return { cp: String.fromCharCode(parseInt(s.slice(pos + 2, pos + 6), 16)), len: 6 };
if (c === "c") {
if (/^[a-zA-Z0-9_]$/.test(s[pos + 2] ?? ""))
return { cp: String.fromCharCode(s[pos + 2].toUpperCase().charCodeAt(0) % 32), len: 3 };
return { cp: undefined, len: 2 };
}
if (c >= "0" && c <= "9") {
if (c === "8" || c === "9")
return { cp: c, len: 2 };
const maxLen = c <= "3" ? 3 : 2;
let j = pos + 1;
while (j < s.length && j - pos - 1 < maxLen && s[j] >= "0" && s[j] <= "7")
j++;
return { cp: String.fromCharCode(parseInt(s.slice(pos + 1, j), 8)), len: j - pos };
}
if (c === "b")
return { cp: "\b", len: 2 };
if (/[dDwWsS]/.test(c))
return { cp: undefined, len: 2 };
if (c === "t")
return { cp: "\t", len: 2 };
if (c === "n")
return { cp: "\n", len: 2 };
if (c === "r")
return { cp: "\r", len: 2 };
if (c === "f")
return { cp: "\f", len: 2 };
if (c === "v")
return { cp: "\v", len: 2 };
return { cp: c, len: 2 };
};
const atomKey = (src) => {
const canon = (ch) => {
if (!ignoreCase)
return ch;
const u = ch.toUpperCase();
if (u.length !== 1)
return ch;
if (ch.charCodeAt(0) >= 128 && u.charCodeAt(0) < 128)
return ch;
return u;
};
const lit = (ch) => `L${canon(ch)}`;
if (src.length === 0)
return src;
if (src[0] !== "\\") {
if (src.length === 1)
return src === "." ? src : lit(src);
if (src[0] === "[" && src.endsWith("]") && src[1] !== "^") {
const body = src.slice(1, -1);
const a1 = readClassAtom(body, 0);
if (a1?.cp !== undefined) {
if (a1.len === body.length)
return lit(a1.cp);
if (body[a1.len] === "-") {
const a2 = readClassAtom(body, a1.len + 1);
if (a2?.cp !== undefined && a1.len + 1 + a2.len === body.length && a1.cp === a2.cp) {
return lit(a1.cp);
}
}
}
return src;
}
if (src.startsWith("(?:") && src.endsWith(")")) {
const body = src.slice(3, -1);
if (body === ".")
return ".";
if (body.length === 1 && !"\\^$.|?*+()[]{}".includes(body))
return lit(body);
if (body[0] === "\\" && escTokenLen(body, 0) === body.length) {
const k = atomKey(body);
if (k.startsWith("L"))
return k;
}
if (body[0] === "[") {
const k = atomKey(body);
if (k.startsWith("L"))
return k;
}
return src;
}
return src;
}
const c = src[1];
if (c !== undefined && c >= "0" && c <= "9") {
const digits = src.slice(1);
if (c !== "0" && Number(digits) <= captureGroups)
return src;
if (c === "8" || c === "9")
return digits.length === 1 ? lit(c) : src;
if (/^[0-7]{1,3}$/.test(digits) && (digits.length < 3 || digits[0] <= "3")) {
return lit(String.fromCharCode(parseInt(digits, 8)));
}
return src;
}
if (c === "x" && src.length === 4)
return lit(String.fromCharCode(parseInt(src.slice(2), 16)));
if (c === "u" && src.length === 6)
return lit(String.fromCharCode(parseInt(src.slice(2), 16)));
if (c === "c" && src.length === 3)
return `L${String.fromCharCode(src[2].toUpperCase().charCodeAt(0) % 32)}`;
if (src.length === 2) {
if (!/[a-zA-Z]/.test(c))
return lit(c);
if (/[dDwWsSbB]/.test(c))
return src;
if (c === "t")
return lit("\t");
if (c === "n")
return lit("\n");
if (c === "r")
return lit("\r");
if (c === "f")
return lit("\f");
if (c === "v")
return lit("\v");
if (c === "c")
return src;
if (c === "k")
return hasNamedGroup ? src : lit("k");
return lit(c);
}
return src;
};
const stack = [{ start: 0, hasUnbounded: false, hasRange: false, hasAlt: false, maxCeil: 1 }];
const cur = () => stack[stack.length - 1];
const atomCtx = [{}];
const atom = () => atomCtx[atomCtx.length - 1];
let lastClosedGroup;
let quantifiedGroup = false;
let unboundedCount = 0;
for (let i = 0; i < pattern.length; i++) {
const c = pattern[i];
const q = quantAt(i);
if (q) {
const lazy = pattern[i + q.len] === "?" ? 1 : 0;
if (q.unbounded)
unboundedCount++;
const g = lastClosedGroup;
const lvl = cur();
const ctx = atom();
if (g) {
quantifiedGroup = true;
const snippet = JSON.stringify(pattern.slice(g.start, i + q.len));
if (q.unbounded) {
if (g.hasUnbounded)
return { risk: `nested unbounded quantifiers in ${snippet} (an unbounded quantifier applied to a group that itself contains one)`, quantifiedGroup, unboundedCount };
if (g.hasAlt)
return { risk: `quantified alternation in ${snippet} (an unbounded quantifier applied to a group containing "|" — the branches can split the same text many ways)`, quantifiedGroup, unboundedCount };
if (g.hasRange)
return { risk: `ambiguous nested repetition in ${snippet} (an unbounded quantifier applied to a group whose own repetition count can vary)`, quantifiedGroup, unboundedCount };
}
else if (q.brace && q.max >= 2) {
if (g.hasUnbounded)
return { risk: `nested quantifiers in ${snippet} (an unbounded quantifier inside a group repeated up to ${q.max} times)`, quantifiedGroup, unboundedCount };
if ((g.hasAlt || g.hasRange) && q.max > REDOS_AMBIGUOUS_REPEAT_CEILING)
return { risk: `ambiguous repetition in ${snippet} (a group with ${g.hasAlt ? "alternation" : "variable-count content"} repeated up to ${q.max} times — candidate paths grow exponentially with the repeat count)`, quantifiedGroup, unboundedCount };
if (g.maxCeil >= 2 && g.maxCeil * q.max > REDOS_BOUNDED_CEILING)
return { risk: `compounded bounded repetition in ${snippet} (inner ceiling ${g.maxCeil} × outer ${q.max} exceeds the ${REDOS_BOUNDED_CEILING} safety ceiling)`, quantifiedGroup, unboundedCount };
}
}
const sideUnbounded = q.unbounded && q.range;
const atomStart = g ? g.start : ctx.lastAtomStart;
if (atomStart !== undefined) {
const atomSrc = pattern.slice(atomStart, i);
const key = atomKey(atomSrc);
const prev = ctx.prevQuant;
if (prev && prev.key === key && prev.unbounded && sideUnbounded) {
const snippet = JSON.stringify(pattern.slice(prev.start, i + q.len + lazy));
return {
risk: `adjacent equivalent quantified atoms in ${snippet} (two consecutive unbounded quantifiers apply to the identical atom ${JSON.stringify(atomSrc)} — every split of a matching run between them is a distinct backtracking path, quadratic in line length on non-matching input; merge them into a single quantifier)`,
quantifiedGroup,
unboundedCount,
};
}
ctx.prevQuant = { key, start: atomStart, unbounded: sideUnbounded };
}
else
ctx.prevQuant = undefined;
if (g) {
if (q.unbounded)
lvl.hasUnbounded = true;
else {
if (q.range)
lvl.hasRange = true;
lvl.maxCeil = Math.max(lvl.maxCeil, Math.max(g.maxCeil, 1) * q.max);
}
}
else if (q.unbounded)
lvl.hasUnbounded = true;
else {
if (q.range)
lvl.hasRange = true;
lvl.maxCeil = Math.max(lvl.maxCeil, q.max);
}
i += q.len + lazy - 1;
ctx.lastAtomStart = undefined;
lastClosedGroup = undefined;
continue;
}
if (lastClosedGroup !== undefined || atom().lastAtomStart !== undefined)
atom().prevQuant = undefined;
lastClosedGroup = undefined;
if (c === "\\") {
atom().lastAtomStart = i;
i += escTokenLen(pattern, i) - 1;
continue;
}
if (c === "[") {
atom().lastAtomStart = i;
i++;
if (pattern[i] === "^")
i++;
while (i < pattern.length && pattern[i] !== "]") {
if (pattern[i] === "\\")
i++;
i++;
}
continue;
}
if (c === "|") {
cur().hasAlt = true;
const ctx = atom();
ctx.lastAtomStart = undefined;
ctx.prevQuant = undefined;
continue;
}
if (c === "(") {
stack.push({ start: i, hasUnbounded: false, hasRange: false, hasAlt: false, maxCeil: 1 });
atomCtx.push({});
if (pattern[i + 1] === "?") {
const c2 = pattern[i + 2];
if (c2 === ":" || c2 === "=" || c2 === "!")
i += 2;
else if (c2 === "<") {
const c3 = pattern[i + 3];
if (c3 === "=" || c3 === "!")
i += 3;
else {
const close = pattern.indexOf(">", i + 3);
if (close > 0)
i = close;
}
}
}
continue;
}
if (c === ")") {
if (stack.length === 1)
continue;
const frame = stack.pop();
atomCtx.pop();
const lvl = cur();
lvl.hasUnbounded ||= frame.hasUnbounded;
lvl.hasRange ||= frame.hasRange;
lvl.hasAlt ||= frame.hasAlt;
lvl.maxCeil = Math.max(lvl.maxCeil, frame.maxCeil);
lastClosedGroup = frame;
atom().lastAtomStart = undefined;
continue;
}
atom().lastAtomStart = i;
}
return { quantifiedGroup, unboundedCount };
}
function redosRejection(risk) {
return (`Error (grep): pattern rejected by the fallback scanner: ${risk}. ` +
`ripgrep is unavailable here, so this pattern would run on a backtracking regex engine where that shape ` +
`can hang the scan (catastrophic backtracking). Rewrite it so a repeated group cannot match the same text ` +
`in more than one way — remove the nested/variable quantifier or the alternation from the repeated group, ` +
`or lower the repetition counts (product ceiling: ${REDOS_BOUNDED_CEILING}).`);
}
const GREP_FILE_BUDGET_MS = 2_000;
const GREP_GRAY_LINE_MAX = 8_192;
const GREP_LINE_CHECK_EVERY = 256;
const monotonicNow = () => (typeof performance !== "undefined" ? performance.now() : Date.now());
class GrepBudgetExceeded extends Error {
}
function compilePattern(p) {
const re = new RegExp(p.pattern, p.ignore_case ? "i" : "");
return (l) => re.test(l);
}
function matchedParts(line, p) {
const parts = [];
const re = new RegExp(p.pattern, `g${p.ignore_case ? "i" : ""}`);
for (let m = re.exec(line); m !== null; m = re.exec(line)) {
if (m[0].length > 0)
parts.push(m[0]);
else
re.lastIndex++;
if (parts.length > 1000)
break;
}
return parts;
}
function multilineSpans(content, p, captureText = false, deadline = Infinity) {
const spans = [];
const starts = [0];
for (let i = 0; i < content.length; i++)
if (content[i] === "\n")
starts.push(i + 1);
const lineAt = (off) => {
let lo = 0, hi = starts.length - 1;
while (lo < hi) {
const mid = (lo + hi + 1) >> 1;
if (starts[mid] <= off)
lo = mid;
else
hi = mid - 1;
}
return lo + 1;
};
const MATCH_TEXT_MAX = 2_000;
const re = new RegExp(p.pattern, `gs${p.ignore_case ? "i" : ""}`);
for (;;) {
if (monotonicNow() >= deadline)
throw new GrepBudgetExceeded();
const m = re.exec(content);
if (m === null)
break;
spans.push([
lineAt(m.index),
lineAt(m.index + Math.max(0, m[0].length - 1)),
captureText ? (m[0].length > MATCH_TEXT_MAX ? `${m[0].slice(0, MATCH_TEXT_MAX)}… [+${m[0].length - MATCH_TEXT_MAX} chars]` : m[0]) : undefined,
]);
if (m[0].length === 0)
re.lastIndex++;
if (spans.length > 10_000)
break;
}
return spans;
}
export async function jsGrepDetailed(env, root, p, signal, guards, deny, denyOut) {
const analysis = analyzeRedos(p.pattern, p.ignore_case === true);
if (analysis.risk)
return { text: redosRejection(analysis.risk) };
const budgetMs = guards?.budgetMs ?? GREP_FILE_BUDGET_MS;
const longLineLimit = guards?.longLineLimit ?? GREP_GRAY_LINE_MAX;
const grayPattern = analysis.quantifiedGroup || analysis.unboundedCount >= 2;
const budgetError = (file, elapsedMs) => `Error (grep): scan stopped after ${Math.round(elapsedMs)}ms inside ${file} — the pattern ` +
`${JSON.stringify(p.pattern)} is backtracking heavily on this content (fallback-engine budget: ` +
`${budgetMs}ms per file). Rewrite the pattern to be more specific, or rerun where ripgrep is available.`;
let skippedLongLines = 0;
let multilineSkippedFiles = 0;
let test;
try {
test = compilePattern(p);
}
catch (e) {
return { text: `Error (grep): invalid regex: ${thrownText(e)}` };
}
const ignore = await buildIgnore(env, root, signal);
const rootPrefix = root.replace(/[\\/]+$/, "") + (root.includes("\\") ? "\\" : "/");
const family = pathFamilyOf({ root });
const start = p.path ? anchorSpelling(family, root, p.path) : root;
let walked;
if (p.path) {
const info = await env.fileInfo(start, signal);
if (info.ok && info.value.kind === "file") {
const oversize = typeof info.value.size === "number" && info.value.size > FILE_MAX_BYTES;
const binary = hasBinaryExtension(start);
walked = {
files: binary || oversize ? [] : [start],
nameOnly: binary || oversize ? [start] : [],
incomplete: false,
skippedLarge: oversize && !binary ? 1 : 0,
skippedUnreadable: 0,
aborted: false,
mtimes: typeof info.value.mtimeMs === "number" ? new Map([[start, info.value.mtimeMs]]) : new Map(),
denyPruned: 0,
denyPatterns: [],
};
}
else {
walked = await walk(env, root, start, ignore, signal, deny);
}
}
else {
walked = await walk(env, root, start, ignore, signal, deny);
}
if (denyOut !== undefined && walked.denyPruned > 0) {
denyOut.withheld = { kind: "pruned_count", count: walked.denyPruned, patterns: walked.denyPatterns };
}
let honestyCaveat = walkCaveat(walked, true);
const denyNote = denyWithheldNote(walked);
let files = walked.files;
files.sort();
const cap = p.head_limit === 0 ? Infinity : Math.max(1, Math.floor(p.head_limit ?? GREP_DEFAULT_CAP));
const mode = p.output_mode ?? "files_with_matches";
const rel = (abs) => (abs.startsWith(rootPrefix) ? abs.slice(rootPrefix.length) : abs);
const startWin = isWinFormPath(start);
const startNorm = collapseDotSegments(start);
const targetSpell = p.path ? p.path.replace(startWin ? /[\\/]+$/ : /\/+$/, "") : "";
const isBoundary = (c) => c === "/" || (startWin && c === "\\");
const relOut = (abs) => {
if (!p.path) {
const r = rel(abs);
return r === abs ? r : `./${r}`;
}
if (abs === startNorm || abs === start)
return targetSpell;
if (abs.startsWith(startNorm) && isBoundary(abs[startNorm.length]))
return targetSpell + abs.slice(startNorm.length);
return rel(abs);
};
if (p.glob) {
const globRes = splitGlobParam(p.glob).map((g) => globTokenToRegExp(normalizeGlobToken(g), globIsAnchored(g), true));
if (globRes.length > 0)
files = files.filter((f) => globRes.some((re) => re.test(rel(f).replace(/\\/g, "/"))));
}
let typeNote = "";
if (p.type) {
const tg = TYPE_GLOBS[p.type.toLowerCase()];
if (tg) {
const typeRe = globTokenToRegExp(tg, false, true);
files = files.filter((f) => typeRe.test(basename(f)));
}
else {
typeNote = `\n[note: unknown type "${p.type}" — searched all files]`;
}
}
const ctxB = Math.max(0, Math.floor(p.context ?? p.context_before ?? 0));
const ctxA = Math.max(0, Math.floor(p.context ?? p.context_after ?? 0));
const ctx = Math.max(ctxB, ctxA);
const off = Math.min(100_000, Math.max(0, Math.floor(p.offset ?? 0)));
const collectCap = cap + off;
const out = [];
const outputMaxBytes = guards?.outputMaxBytes ?? JS_GREP_OUTPUT_MAX_BYTES;
let outBytes = 0;
let outputTruncated = false;
let rowsBeforeWindow = 0;
const pushRow = (row, path) => {
if (outputTruncated)
return;
if (rowsBeforeWindow < off) {
rowsBeforeWindow++;
return;
}
if (out.length >= cap)
return;
const size = Buffer.byteLength(row, "utf8") + 1;
if (outBytes + size > outputMaxBytes) {
outputTruncated = true;
return;
}
outBytes += size;
out.push({ path, text: row });
};
const fileMatches = [];
const counts = [];
let totalContent = 0;
let totalFiles = 0;
let skippedUnreadableFiles = 0;
let scanAborted = false;
let scanned = 0;
for (const f of files) {
if (++scanned % SCAN_YIELD_EVERY === 0)
await yieldEventLoop();
if (signal?.aborted) {
scanAborted = true;
break;
}
const read = await env.readTextFile(f, signal);
if (!read.ok) {
if (signal?.aborted) {
scanAborted = true;
break;
}
skippedUnreadableFiles++;
continue;
}
const lines = read.value.split("\n");
if (lines.length > 1 && lines.at(-1) === "")
lines.pop();
const shownPath = relOut(f);
let fileCount = 0;
const fileMatchStart = monotonicNow();
if (p.multiline) {
if (grayPattern && read.value.length > longLineLimit) {
multilineSkippedFiles++;
continue;
}
let spans;
try {
spans = multilineSpans(read.value, p, mode === "content" && p.only_matching === true, fileMatchStart + budgetMs);
}
catch (e) {
if (e instanceof GrepBudgetExceeded)
return { text: budgetError(shownPath, monotonicNow() - fileMatchStart) };
return { text: `Error (grep): invalid regex: ${thrownText(e)}` };
}
fileCount = spans.length;
if (mode === "content") {
for (const [s, eL, text] of spans) {
if (p.only_matching && text !== undefined) {
const parts = text.split("\n");
for (let k = 0; k < parts.length; k++) {
pushRow(`${shownPath}:${s + k}:${clipLine(parts[k])}`, shownPath);
}
continue;
}
for (let j = Math.max(0, s - 1 - ctxB); j <= Math.min(lines.length - 1, eL - 1 + ctxA); j++) {
pushRow(`${shownPath}:${j + 1}:${clipLine(lines[j])}`, shownPath);
}
}
}
}
else {
for (let i = 0; i < lines.length; i++) {
if (grayPattern || (i > 0 && i % GREP_LINE_CHECK_EVERY === 0)) {
const elapsed = monotonicNow() - fileMatchStart;
if (elapsed >= budgetMs)
return { text: budgetError(shownPath, elapsed) };
}
if (i > 0 && i % GREP_LINE_CHECK_EVERY === 0) {
await yieldEventLoop();
if (signal?.aborted) {
scanAborted = true;
break;
}
}
if (grayPattern && lines[i].length > longLineLimit) {
skippedLongLines++;
continue;
}
if (!test(lines[i]))
continue;
fileCount++;
if (mode === "content") {
if (p.only_matching) {
for (const part of matchedParts(lines[i], p)) {
pushRow(`${shownPath}:${i + 1}:${clipLine(part)}`, shownPath);
}
}
else if (ctx > 0) {
for (let j = Math.max(0, i - ctxB); j <= Math.min(lines.length - 1, i + ctxA); j++) {
pushRow(`${shownPath}:${j + 1}:${clipLine(lines[j])}`, shownPath);
}
}
else {
pushRow(`${shownPath}:${i + 1}:${clipLine(lines[i])}`, shownPath);
}
}
}
}
if (scanAborted)
break;
if (mode === "content")
totalContent += fileCount;
if (fileCount > 0) {
totalFiles++;
if (mode === "files_with_matches")
fileMatches.push(f);
else if (mode === "count" && counts.length < collectCap)
counts.push({ path: shownPath, text: `${shownPath}:${fileCount}` });
}
}
if (scanAborted || skippedUnreadableFiles > 0 || skippedLongLines > 0 || multilineSkippedFiles > 0) {
const extra = [];
if (scanAborted && !walked.aborted)
extra.push("the search was interrupted before finishing");
if (skippedUnreadableFiles > 0)
extra.push(`${skippedUnreadableFiles} file(s) could not be read and were skipped`);
if (skippedLongLines > 0)
extra.push(`${skippedLongLines} line(s) over ${longLineLimit} chars were not matched (the pattern's worst-case matching cost grows superlinearly with line length on the fallback engine)`);
if (multilineSkippedFiles > 0)
extra.push(`${multilineSkippedFiles} file(s) skipped by multiline matching (content over ${longLineLimit} chars; the pattern's worst-case matching cost grows superlinearly with input length on the fallback engine)`);
if (extra.length > 0) {
honestyCaveat = honestyCaveat.length > 0
? honestyCaveat.replace(/\]$/, `; ${extra.join("; ")}]`)
: `\n…[results may be incomplete: ${extra.join("; ")}]`;
}
}
const caveat = honestyCaveat + denyNote;
const paged = (arr) => (off > 0 ? arr.slice(off, off + cap) : [...arr]);
const offNote = off > 0 ? `\n[offset ${off}]` : "";
if (mode === "files_with_matches") {
const haveMtimes = fileMatches.length > 0 && fileMatches.every((f) => walked.mtimes.has(f));
const sortedFileMatches = haveMtimes
? fileMatches
.map((f) => ({ r: relOut(f), m: walked.mtimes.get(f) }))
.sort((a, b) => b.m - a.m || (a.r < b.r ? -1 : a.r > b.r ? 1 : 0))
.map((x) => x.r)
: fileMatches.map(relOut).sort();
const body = sortedFileMatches.slice(off, off + cap);
const rows = body.map((r) => ({ path: r, text: r }));
return {
text: body.length === 0
? NO_MATCHES + offNote + typeNote + caveat
: body.join("\n") + (totalFiles > off + cap ? `\n…[capped at ${cap} of ${totalFiles}]` : "") + offNote + typeNote + caveat,
rows,
};
}
if (mode === "count") {
const body = paged(counts);
return {
text: body.length === 0
? NO_MATCHES + offNote + typeNote + caveat
: body.map((r) => r.text).join("\n") + (totalFiles > off + cap ? `\n…[capped at ${cap} of ${totalFiles}]` : "") + offNote + typeNote + caveat,
rows: body,
};
}
const body = out;
const byteNote = outputTruncated ? `\n…[output truncated at ${outputMaxBytes} bytes — narrow the pattern or set a smaller head_limit]` : "";
if (body.length === 0)
return { text: NO_MATCHES + byteNote + offNote + typeNote + caveat, rows: [] };
const bodyText = body.map((r) => r.text).join("\n");
if (out.length < cap)
return { text: bodyText + byteNote + offNote + typeNote + caveat, rows: body };
const marker = ctx > 0 || p.multiline || p.only_matching
? `\n…[capped at ${cap}; ${totalContent}+ match(es) found, more output omitted]`
: `\n…[capped at ${cap} of ${totalContent}]`;
return { text: bodyText + marker + byteNote + offNote + typeNote + caveat, rows: body };
}
export async function jsGrep(env, root, p, signal, guards, deny, denyOut) {
return (await jsGrepDetailed(env, root, p, signal, guards, deny, denyOut)).text;
}
const rgCache = new WeakMap();
export function detectRipgrep(env) {
let cached = rgCache.get(env);
if (!cached) {
cached = env
.exec("rg --version", { timeout: 10 })
.then((r) => r.ok && r.value.exitCode === 0)
.catch(() => false);
rgCache.set(env, cached);
}
return cached;
}
async function sortRgFilesByMtime(env, root, records, signal) {
if (records.length === 0)
return [...records];
const family = pathFamilyOf({ root });
const toAbs = (p) => {
const stripped = p.startsWith("./") ? p.slice(2) : p;
return anchorSpelling(family, root, stripped);
};
const paths = records.map((r) => r.path ?? r.text);
const infos = await Promise.all(paths.map((p) => env.fileInfo(toAbs(p), signal)));
const withMtime = records.map((r, i) => {
const info = infos[i];
return { r, p: paths[i], m: info.ok && typeof info.value.mtimeMs === "number" ? info.value.mtimeMs : undefined };
});
const sorted = withMtime.every((x) => x.m !== undefined)
? [...withMtime].sort((a, b) => b.m - a.m || (a.p < b.p ? -1 : a.p > b.p ? 1 : 0))
: [...withMtime].sort((a, b) => (a.p < b.p ? -1 : a.p > b.p ? 1 : 0));
return sorted.map((x) => x.r);
}
function rgRecordSeparator(mode, rest) {
if (mode === "count")
return /^\d+$/.test(rest) ? ":" : null;
const m = /^\d+([:\-])/.exec(rest);
return m === null ? null : m[1];
}
function parseRgRecords(stdout, mode, dropIncompleteTail = false) {
if (stdout.length === 0)
return [];
if (mode === "files_with_matches") {
if (!stdout.includes("\0")) {
if (dropIncompleteTail)
return [];
return stdout.split("\n").filter((l) => l.length > 0).map((l) => ({ path: l, text: l }));
}
const parts = stdout.split("\0");
const tail = parts.pop() ?? "";
const records = parts.filter((p) => p.length > 0).map((p) => ({ path: p, text: p }));
if (tail.length > 0 && !dropIncompleteTail)
records.push({ text: tail });
return records;
}
const lines = stdout.split("\n");
if (!stdout.endsWith("\n") && dropIncompleteTail)
lines.pop();
const records = [];
for (const line of lines) {
if (line.length === 0)
continue;
const nul = line.indexOf("\0");
const rest = nul < 0 ? "" : line.slice(nul + 1);
const sep = nul < 0 ? null : rgRecordSeparator(mode, rest);
if (nul < 0 || sep === null) {
records.push({ text: line });
continue;
}
const path = line.slice(0, nul);
records.push({ path, text: `${path}${sep}${rest}` });
}
return records;
}
function formatRgRecords(records, p, caveat = "") {
const cap = p.head_limit === 0 ? Infinity : Math.max(1, Math.floor(p.head_limit ?? GREP_DEFAULT_CAP));
const off = Math.max(0, Math.floor(p.offset ?? 0));
const capped = records.slice(off, off + cap);
const served = capped.map((r) => (r.path !== undefined ? { path: r.path, text: r.text } : { text: r.text }));
if (capped.length === 0)
return { text: NO_MATCHES + (off > 0 ? `\n[offset ${off}]` : "") + caveat, served };
return {
text: capped.map((r) => r.text).join("\n") +
(records.length > off + cap ? `\n…[capped at ${cap} of ${records.length}]` : "") +
(off > 0 ? `\n[offset ${off}]` : "") +
caveat,
served,
};
}
export function rgOutputDenyTripwire(stdout, mode, judge, opts = {}) {
if (stdout.length === 0)
return { trip: false };
const absolute = rgRecordAbsolute(opts.base);
for (const record of parseRgRecords(stdout, mode, opts.dropIncompleteTail === true)) {
if (record.path === undefined) {
if (record.text === "--")
continue;
return { trip: true, reason: "ambiguous-record" };
}
const hit = judge.matchPath(absolute(record.path));
if (hit !== null)
return { trip: true, reason: "deny-hit", pattern: hit.pattern };
}
return { trip: false };
}
function rgRecordAbsolute(base) {
if (base === undefined)
return (p) => p;
const family = pathFamilyOf({ root: base });
return (p) => anchorSpelling(family, base, p.replace(/^\.[\\/]/, ""));
}
async function rgOutputDenyTripwireAt(env, stdout, mode, deny, opts, signal) {
const views = homeViewsOf(env, deny, signal);
const absolute = rgRecordAbsolute(opts.base);
const placed = new Map();
for (const record of parseRgRecords(stdout, mode, opts.dropIncompleteTail === true)) {
if (record.path === undefined)
continue;
const path = absolute(record.path);
if (placed.has(path))
continue;
const home = await views.of(path, { leaf: "file" });
if (home.view !== undefined)
placed.set(path, home.view);
if (home.unread !== undefined && deny.matchPath(path) === null && (home.view === undefined || deny.matchPath(home.view) === null))
throw new HomeViewUnreadError(path, home.unread);
}
const judge = placed.size === 0 ? deny : { matchPath: (p) => deny.matchPath(p) ?? (placed.has(p) ? deny.matchPath(placed.get(p)) : null) };
return rgOutputDenyTripwire(stdout, mode, judge, opts);
}
async function jsGrepDenyTripFallback(env, root, p, signal, trip, deny) {
const denyOut = {};
const { text, rows } = await jsGrepDetailed(env, root, p, signal, undefined, deny, denyOut);
const why = trip.reason === "deny-hit" ? "its output involved sensitive-path deny-listed entries the exclusion globs cannot express" : "its output contained a record the line format cannot account for";
return {
text: text.startsWith("Error (grep)")
? text
: `${text}\n[note: the ripgrep pass was abandoned (${why}); results are from the fallback scanner, which prunes with the authoritative deny judge]`,
...(rows !== undefined ? { rows } : {}),
degraded: { fallback: "js-scan", reason: `ripgrep output tripped the deny tripwire (${trip.reason})` },
...(denyOut.withheld !== undefined ? { withheld: denyOut.withheld } : {}),
};
}
async function jsGrepFallback(env, root, p, signal, reason, deny) {
const denyOut = {};
const { text, rows } = await jsGrepDetailed(env, root, p, signal, undefined, deny, denyOut);
return {
text: text.startsWith("Error (grep)")
? text
: `${text}\n[note: ripgrep failed (${reason}); results are from a fallback scanner]`,
...(rows !== undefined ? { rows } : {}),
degraded: { fallback: "js-scan", reason: `ripgrep failed (${reason})` },
...(denyOut.withheld !== undefined ? { withheld: denyOut.withheld } : {}),
};
}
export async function rgGrepDetailed(env, root, p, signal, deny) {
const mode = p.output_mode ?? "files_with_matches";
const flags = ["--null", "-H", "--no-messages", "--no-require-git", "--hidden"];
for (const d of VCS_DIRS)
flags.push("--glob", `!${d}`);
flags.push("--max-columns", "500", "--max-columns-preview");
if (p.ignore_case)
flags.push("-i");
if (p.multiline)
flags.push("-U", "--multiline-dotall");
if (mode === "content") {
flags.push("--no-heading", "-n");
if (p.only_matching)
flags.push("-o");
const ctxB = p.context ?? p.context_before;
const ctxA = p.context ?? p.context_after;
if (ctxB && ctxB > 0)
flags.push("-B", String(Math.floor(ctxB)));
if (ctxA && ctxA > 0)
flags.push("-A", String(Math.floor(ctxA)));
}
else if (mode === "files_with_matches")
flags.push("-l");
else if (mode === "count")
flags.push("-c");
if (p.glob)
for (const g of splitGlobParam(p.glob))
flags.push("--glob", shellQuote(g));
if (p.type)
flags.push("--type", shellQuote(p.type));
if (deny !== undefined)
for (const g of deny.rgExclusionGlobs)
flags.push(g.flag, shellQuote(g.glob));
const target = p.path ? shellQuote(p.path) : ".";
const cmd = `rg ${flags.join(" ")} -e ${shellQuote(p.pattern)} -- ${target}`;
const res = await env.exec(cmd, { cwd: root, timeout: 60, abortSignal: signal });
if (!res.ok) {
const err = res.error;
if (err.code === "aborted" || signal?.aborted === true)
throw err;
if (isTimeCutCode(err.code)) {
const partial = err.partialStdout ?? "";
if (partial.trim().length > 0) {
if (deny !== undefined) {
const trip = await rgOutputDenyTripwireAt(env, partial, mode, deny, { dropIncompleteTail: true, base: root }, signal);
if (trip.trip)
return jsGrepDenyTripFallback(env, root, p, signal, trip, deny);
}
const records = parseRgRecords(partial, mode, true);
if (records.length === 0)
return jsGrepFallback(env, root, p, signal, "timed out before completing a result", deny);
return {
...(() => { const f = formatRgRecords(records, p); return { text: `${delimitUntrusted("ripgrep partial output", f.text)}\n…[ripgrep timed out after producing partial output — results may be incomplete]`, rows: f.served }; })(),
degraded: { partial: true, reason: "ripgrep timed out" },
};
}
return jsGrepFallback(env, root, p, signal, "timed out with no output", deny);
}
return jsGrepFallback(env, root, p, signal, `could not run: ${err.code}`, deny);
}
const { exitCode, stdout } = res.value;
const denyDisclosure = async () => {
if (deny === undefined || deny.rgExclusionGlobs.length === 0)
return { note: "" };
if (p.path) {
const startPath = anchorSpelling(pathFamilyOf({ root }), root, p.path);
const info = await env.fileInfo(startPath, signal);
if (info.ok && info.value.kind === "file")
return { note: "" };
}
const withheld = await rgDenyExistenceProbe(env, root, deny, target, signal);
if (withheld === undefined)
return { note: "" };
const note = withheld.kind === "existence"
? "\n[note: entries matching the sensitive-path read deny list exist under this scope and were excluded (readDenyPatterns)]"
: "\n[note: entries matching the sensitive-path read deny list may have been excluded (deny-existence probe failed)]";
return { note, withheld };
};
if (exitCode === 1) {
const d = await denyDisclosure();
return { text: NO_MATCHES + d.note, rows: [], ...(d.withheld !== undefined ? { withheld: d.withheld } : {}) };
}
if (exitCode >= 2) {
if (stdout.trim().length > 0) {
if (deny !== undefined) {
const trip = await rgOutputDenyTripwireAt(env, stdout, mode, deny, { dropIncompleteTail: true, base: root }, signal);
if (trip.trip)
return jsGrepDenyTripFallback(env, root, p, signal, trip, deny);
}
const records = parseRgRecords(stdout, mode, true);
if (records.length === 0)
return jsGrepFallback(env, root, p, signal, `exited with code ${exitCode} and produced no complete result`, deny);
return {
...(() => { const f = formatRgRecords(records, p); return { text: `${delimitUntrusted("ripgrep partial output", f.text)}\n…[ripgrep exited with an error after producing partial output — results may be incomplete]`, rows: f.served }; })(),
degraded: { partial: true, reason: `ripgrep exited with code ${exitCode}` },
};
}
return jsGrepFallback(env, root, p, signal, `exited with code ${exitCode} and produced no output`, deny);
}
if (deny !== undefined) {
const trip = await rgOutputDenyTripwireAt(env, stdout, mode, deny, { base: root }, signal);
if (trip.trip)
return jsGrepDenyTripFallback(env, root, p, signal, trip, deny);
}
const parsed = parseRgRecords(stdout, mode);
const ordered = mode === "files_with_matches" ? await sortRgFilesByMtime(env, root, parsed, signal) : parsed;
const d = await denyDisclosure();
const formatted = formatRgRecords(ordered, p);
return { text: formatted.text + d.note, rows: formatted.served, ...(d.withheld !== undefined ? { withheld: d.withheld } : {}) };
}
async function rgDenyExistenceProbe(env, root, deny, target, signal) {
const probeFlags = ["--files", "--hidden", "--no-require-git", "--no-messages"];
for (const d of VCS_DIRS)
probeFlags.push("--glob", `!${d}`);
for (const g of deny.rgProbeGlobs)
probeFlags.push(g.flag, shellQuote(g.glob));
try {
const res = await env.exec(`( rg ${probeFlags.join(" ")} -- ${target}; echo "__RG_PROBE_EXIT_$?" 1>&2 ) | head -n 1`, { cwd: root, timeout: 10, abortSignal: signal });
if (!res.ok)
return { kind: "probe_failed" };
const first = res.value.stdout.split("\n").find((l) => l.trim().length > 0);
if (first !== undefined) {
if (/^__RG_PROBE_EXIT_\d+$/.test(first.trim()))
return { kind: "probe_failed" };
const hit = deny.matchPath(first.trim());
return { kind: "existence", ...(hit !== null ? { patterns: [hit.pattern] } : {}) };
}
const marker = /__RG_PROBE_EXIT_(\d+)/.exec(res.value.stderr ?? "");
const rgExit = marker === null ? undefined : Number(marker[1]);
if (rgExit === 1)
return undefined;
if (rgExit === 0)
return undefined;
return { kind: "probe_failed" };
}
catch {
return { kind: "probe_failed" };
}
}
export async function runGrepDetailed(env, root, p, signal, deny) {
const scope = p.path ? anchorSpelling(pathFamilyOf({ root }), root, p.path) : root;
const unexcludableUnder = async () => {
if (deny === undefined)
return false;
const home = await homeViewsOf(env, deny, signal).of(scope);
if (home.unread !== undefined)
throw new HomeViewUnreadError(scope, home.unread);
if (deny.unexcludableUnder === undefined)
return false;
return deny.unexcludableUnder(scope) || (home.view !== undefined && deny.unexcludableUnder(home.view));
};
if (!(await unexcludableUnder()) && (await detectRipgrep(env)))
return rgGrepDetailed(env, root, p, signal, deny);
const denyOut = {};
const { text, rows } = await jsGrepDetailed(env, root, p, signal, undefined, deny, denyOut);
return { text, ...(rows !== undefined ? { rows } : {}), ...(denyOut.withheld !== undefined ? { withheld: denyOut.withheld } : {}) };
}
export async function runGrep(env, root, p, signal) {
return (await runGrepDetailed(env, root, p, signal)).text;
}
export async function runGlob(env, root, pattern, opts = {}, signal) {
return (await runGlobDetailed(env, root, pattern, opts, signal)).text;
}
export async function runGlobDetailed(env, root, pattern, opts = {}, signal) {
const t0 = Date.now();
const baseIgnore = await buildIgnore(env, root, signal);
const rootPrefix = root.replace(/[\\/]+$/, "") + (root.includes("\\") ? "\\" : "/");
const family = pathFamilyOf({ root });
const start = opts.path ? anchorSpelling(family, root, opts.path) : root;
if (opts.path !== undefined) {
const probe = await env.fileInfo(start, signal);
if (!probe.ok && probe.error.code === "not_found") {
const error = `Error (Glob): path ${JSON.stringify(opts.path)} does not exist — check the path, or omit \`path\` to search the whole root.`;
return { text: error, error, filenames: [], numFiles: 0, truncated: false, durationMs: Date.now() - t0, totalMatches: 0, countIsComplete: false };
}
if (probe.ok && probe.value.kind === "file") {
const error = `Error (Glob): path ${JSON.stringify(opts.path)} is not a directory — \`path\` scopes the search to a directory; pass its parent directory (and name the file in \`pattern\`), or omit \`path\` to search the whole root.`;
return { text: error, error, filenames: [], numFiles: 0, truncated: false, durationMs: Date.now() - t0, totalMatches: 0, countIsComplete: false };
}
}
const pat = normalizeGlobToken(pattern);
const anchored = globIsAnchored(pattern);
const re = globTokenToRegExp(pat, anchored, true);
const startPrefix = start.replace(/[\\/]+$/, "") + (start.includes("\\") ? "\\" : "/");
const startRelPrefix = startPrefix.startsWith(rootPrefix) ? startPrefix.slice(rootPrefix.length) : startPrefix;
const toStartRel = (p) => {
const s = p.startsWith(startPrefix)
? p.slice(startPrefix.length)
: startRelPrefix.length > 0 && p.startsWith(startRelPrefix)
? p.slice(startRelPrefix.length)
: p;
return s.replace(/\\/g, "/");
};
const patternSegs = pat.split("/");
const groupSpansSlash = /\{[^}]*\/[^}]*\}|\[[^\]]*\/[^\]]*\]/.test(pat);
const prunable = anchored && !groupSpansSlash;
const firstStarStar = patternSegs.findIndex((s) => s.includes("**"));
const starStarIdx = firstStarStar === -1 ? Infinity : firstStarStar;
const segRes = prunable ? patternSegs.map((s) => globTokenToRegExp(s, true, true)) : [];
const isLiteralSeg = (s) => s.length > 0 && !/[*?[{]/.test(s);
const literalAt = prunable ? patternSegs.map((s) => (isLiteralSeg(s) ? s : undefined)) : [];
const literalAfterStarStar = new Set(prunable ? patternSegs.filter((s, i) => i > starStarIdx && isLiteralSeg(s)) : []);
const mayContainMatch = (dirRel) => {
if (!prunable)
return true;
const d = dirRel.split("/");
if (d.length >= patternSegs.length && starStarIdx === Infinity)
return false;
const lim = Math.min(d.length, starStarIdx);
for (let i = 0; i < lim; i++)
if (!segRes[i].test(d[i]))
return false;
return true;
};
const explicitlyNamed = (sr) => {
if (!prunable)
return false;
const segs = sr.split("/");
const k = segs.length - 1;
return k < starStarIdx ? literalAt[k] === segs[k] : literalAfterStarStar.has(segs[k]);
};
let ignoredDirs = 0;
let ignoredFiles = 0;
const rescued = [];
const ignore = (relPath, isDir) => {
const sr = toStartRel(relPath);
if (isDir && !mayContainMatch(sr))
return true;
if (explicitlyNamed(sr)) {
if (isDir && baseIgnore(relPath, true))
rescued.push(sr + "/");
return false;
}
if (rescued.length > 0 && rescued.some((r) => sr.startsWith(r)))
return false;
const ig = baseIgnore(relPath, isDir);
if (ig) {
if (isDir)
ignoredDirs++;
else
ignoredFiles++;
}
return ig;
};
const walked = await walk(env, root, start, ignore, signal, opts.deny);
const caveat = walkCaveat(walked, false) + denyWithheldNote(walked);
const files = [...walked.files, ...walked.nameOnly];
const rel = (abs) => (abs.startsWith(rootPrefix) ? abs.slice(rootPrefix.length) : abs);
const cap = Math.max(1, Math.floor(opts.max ?? 500));
const matchedAbs = files.filter((f) => re.test(toStartRel(f)) || (!anchored && re.test(basename(f))));
const haveMtimes = matchedAbs.length > 0 && matchedAbs.every((f) => walked.mtimes.has(f));
const matchedAll = haveMtimes
? matchedAbs
.map((f) => ({ r: rel(f), m: walked.mtimes.get(f) }))
.sort((a, b) => b.m - a.m || (a.r < b.r ? -1 : a.r > b.r ? 1 : 0))
.map((x) => x.r)
: matchedAbs.map(rel).sort();
const matched = matchedAll.slice(0, cap);
const totalMatches = matchedAll.length;
const countIsComplete = !walkIsPartial(walked);
const truncated = !countIsComplete || totalMatches > cap;
const capNote = !truncated || matched.length === 0
? ""
: countIsComplete
? `\n(Showing ${matched.length} of ${totalMatches} matching files; ${totalMatches - matched.length} more are not listed. Narrow the pattern or path to see the rest.)`
: `\n(Showing the first ${matched.length} files; there are more than ${totalMatches} matches. Narrow the pattern or path to see the rest.)`;
const ignoreNote = matched.length === 0 && ignoredDirs > 0
? `\n[note: ${ignoredDirs} ignored director${ignoredDirs === 1 ? "y was" : "ies were"} not searched (dependency/build/VCS trees and .gitignore) — name a directory in the pattern (e.g. "dist/**") or pass \`path\` to include it]`
: "";
const ignoredFileNote = matched.length === 0 && ignoredFiles > 0
? `\n[note: ${ignoredFiles} file(s) matching an ignore rule (.gitignore) were skipped — an empty result here is NOT proof the file is absent; name it literally with a root-anchored path (e.g. "/notes.secret", "logs/app.log") to include it, or read it directly]`
: "";
const budgetNote = matched.length === 0 && walked.incomplete
? `\n[note: the file-walk budget (${WALK_MAX_FILES} files / ${WALK_MAX_DEPTH} directory levels) ran out before the whole tree was scanned — this empty result is NOT proof the file is absent; scope the search with \`path\`, or use a root-anchored pattern (e.g. "src/**/*.ts") so the walk is pruned toward it, then retry]`
: "";
const text = matched.length === 0 ? "No files matched." + ignoreNote + ignoredFileNote + budgetNote + caveat : matched.join("\n") + capNote + caveat;
return {
text,
filenames: matched,
numFiles: matched.length,
truncated,
durationMs: Date.now() - t0,
totalMatches,
countIsComplete,
...(walked.denyPruned > 0 ? { withheld: { kind: "pruned_count", count: walked.denyPruned, patterns: walked.denyPatterns } } : {}),
};
}