@sema-agent/core
Version:
Stateless, task-oriented AI agent core
1,207 lines (1,206 loc) • 45.5 kB
JavaScript
export const SHELL_LEX_DIALECT = "bash";
export const QUOTE_KINDS = Object.freeze(["'", '"', "$'", '$"', "\\"]);
export const EXPANSION_KINDS = Object.freeze(["parameter", "command", "process", "arithmetic", "tilde", "brace", "glob"]);
export const UNRESOLVED_REASONS = Object.freeze(["ansi_c_escape", "locale_string", "parameter_text", "subscript_text", "compound_array_value"]);
export const UNRESOLVED_SENTENCE = {
ansi_c_escape: "an ANSI-C quoted word carries an escape this lexer does not fold",
locale_string: "a locale-translated string carries text this lexer does not read",
parameter_text: "a parameter expansion carries text this lexer does not read",
subscript_text: "a subscript carries text this lexer does not read",
compound_array_value: "a compound assignment value is a list this lexer does not read",
};
export const PARSE_FAILURE_REASONS = Object.freeze([
"unterminated_single_quote",
"unterminated_double_quote",
"unterminated_ansi_c",
"unterminated_subscript",
"unterminated_compound_value",
"unterminated_command_substitution",
"unterminated_process_substitution",
"unterminated_parameter_expansion",
"unterminated_heredoc",
"undelimited_substitution",
"redirect_without_target",
"heredoc_without_delimiter",
"heredoc_delimiter_expands",
]);
export const PARSE_FAILURE_SENTENCE = {
unterminated_single_quote: "a single-quoted word never closes",
unterminated_double_quote: "a double-quoted word never closes",
unterminated_ansi_c: "an ANSI-C quoted word never closes",
unterminated_subscript: "a subscript never closes",
unterminated_compound_value: "a compound assignment value never closes",
unterminated_command_substitution: "a command substitution never terminates",
unterminated_process_substitution: "a process substitution never terminates",
unterminated_parameter_expansion: "a parameter expansion never terminates",
unterminated_heredoc: "a here-document body never terminates",
undelimited_substitution: "a command substitution carries a construct this lexer cannot delimit",
redirect_without_target: "a redirection names no target",
heredoc_without_delimiter: "a here-document operator names no delimiter",
heredoc_delimiter_expands: "a here-document delimiter is not a plain word",
};
export const BASH_ONLY_CONSTRUCTS = Object.freeze([
"ansi_c_quoting",
"locale_string",
"append_assignment",
"subscript_assignment",
"compound_assignment",
"here_string",
"process_substitution",
"redirect_both",
"pipe_both",
"brace_expansion",
]);
export const ANSI_C_ESCAPES = {
a: "\x07",
b: "\b",
e: "\x1b",
E: "\x1b",
f: "\f",
n: "\n",
r: "\r",
t: "\t",
v: "\v",
"\\": "\\",
"'": "'",
'"': '"',
"?": "?",
};
export const SUBSTITUTION_PLACEHOLDER = " subst";
export const SHELL_CONTROL_PREFIXES = new Set(["!", "if", "then", "elif", "else", "while", "until", "do", "time"]);
const COMPOUND_OPERAND_COMMANDS = new Set(["alias", "declare", "eval", "export", "let", "local", "readonly", "typeset"]);
const GROUP_WORDS = new Set(["{", "}"]);
export function dollarOpensQuote(text, at) {
const next = text[at + 1];
if (next !== "'" && next !== '"')
return false;
let run = 0;
for (let k = at - 1; k >= 0 && text[k] === "\\"; k--)
run += 1;
return run % 2 === 0;
}
export function ansiCQuoteClose(source, open) {
for (let i = open + 1; i < source.length; i++) {
if (source[i] === "\\") {
i += 1;
continue;
}
if (source[i] === "'")
return i;
}
return undefined;
}
export function literalTildeSpelling(word) {
return word.startsWith("~") ? `./${word}` : word;
}
function decodeAnsiC(body) {
let text = "";
let unresolved = false;
for (let i = 0; i < body.length; i++) {
const c = body[i];
if (c !== "\\" || i + 1 >= body.length) {
text += c;
continue;
}
const next = body[i + 1];
const folded = ANSI_C_ESCAPES[next];
if (folded === undefined) {
unresolved = true;
text += c + next;
}
else
text += folded;
i++;
}
return { text, unresolved };
}
export function argvOf(segment) {
return segment.tokens.filter((t) => t.kind === "word");
}
export function wordsOf(segment) {
const folded = [];
const raw = [];
for (const t of segment.tokens) {
if (t.kind !== "word")
continue;
folded.push(t.folded);
raw.push(t.raw);
}
return { folded, raw };
}
function past(text, j) {
let k = j;
while (text[k] === "\\" && text[k + 1] === "\n")
k += 2;
return k;
}
function parenBodyEnd(text, open) {
let depth = 0;
let quoted;
let wordStart = true;
let word = "";
for (let j = open; j < text.length; j++) {
const c = text[j];
if (quoted !== undefined) {
if (c === "\\" && quoted === '"')
j++;
else if (c === quoted)
quoted = undefined;
wordStart = false;
continue;
}
if (c === "\\") {
if (text[j + 1] === "\n") {
j++;
continue;
}
j++;
wordStart = false;
word = "";
continue;
}
if (c === "#" && wordStart) {
const nl = text.indexOf("\n", j);
if (nl < 0)
return undefined;
j = nl;
wordStart = true;
word = "";
continue;
}
if (c === "'" || c === '"') {
quoted = c;
wordStart = false;
word = "";
continue;
}
if (c === "$" && text[past(text, j + 1)] === "{") {
const open2 = past(text, j + 1);
const close = braceExpansionEnd(text, open2);
if (close === undefined)
return undefined;
if (UNREADABLE_EXPANSION_TEXT.test(text.slice(open2 + 1, close - 1)))
return "unreadable";
j = close - 1;
wordStart = false;
word = "";
continue;
}
if (c === "<" && text[past(text, j + 1)] === "<" && text[past(text, past(text, j + 1) + 1)] !== "<")
return "unreadable";
if (c === "(")
depth++;
else if (c === ")") {
depth--;
if (depth === 0)
return j + 1;
}
const boundary = /[\s;|&()]/.test(c);
if (boundary && word === "case")
return "unreadable";
word = boundary ? "" : word + c;
wordStart = boundary;
}
return undefined;
}
function braceExpansionEnd(text, open) {
let quoted;
for (let j = open + 1; j < text.length; j++) {
const c = text[j];
if (quoted !== undefined) {
if (c === "\\" && quoted === '"')
j++;
else if (c === quoted)
quoted = undefined;
continue;
}
if (c === "\\") {
j++;
continue;
}
if (c === "'" || c === '"') {
quoted = c;
continue;
}
if (c === "}")
return j + 1;
}
return undefined;
}
function matchedParen(text, open) {
let depth = 0;
let quoted;
for (let j = open; j < text.length; j++) {
const c = text[j];
if (quoted !== undefined) {
if (c === "\\" && quoted === '"')
j++;
else if (c === quoted)
quoted = undefined;
continue;
}
if (c === "\\") {
j++;
continue;
}
if (c === "'" || c === '"') {
quoted = c;
continue;
}
if (c === "$" && text[j + 1] === "'") {
const close = ansiCQuoteClose(text, j + 1);
if (close === undefined)
return undefined;
j = close;
continue;
}
if (c === "$" && text[j + 1] === "{") {
const close = braceExpansionEnd(text, j + 1);
if (close === undefined)
return undefined;
j = close - 1;
continue;
}
if (c === "(")
depth++;
else if (c === ")" && --depth === 0)
return j + 1;
}
return undefined;
}
const UNREADABLE_EXPANSION_TEXT = /[$`()<>{\\]/;
const EXPANSION_NAME = /^[#!]?([A-Za-z_][A-Za-z0-9_]*|\d+|[@*#?$!0-])/;
const BARE_PARAMETER = /^(?:[A-Za-z_][A-Za-z0-9_]*|\d|[@*#?$!0-])/;
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
const NAME_HEAD = /^[A-Za-z_][A-Za-z0-9_]*/;
function backtickBody(source, open) {
let body = "";
for (let j = open + 1; j < source.length; j++) {
const c = source[j];
if (c === "\\" && j + 1 < source.length) {
const next = source[j + 1];
if (next === "\n") {
j++;
continue;
}
body += next === "$" || next === "`" || next === "\\" ? next : c + next;
j++;
continue;
}
if (c === "`")
return { body, close: j };
body += c;
}
return undefined;
}
export function substitutionsInHeredocBody(body) {
const out = [];
for (let i = 0; i < body.length; i++) {
const ch = body[i];
if (ch === "\\") {
i++;
continue;
}
if (ch === "`") {
const tick = backtickBody(body, i);
if (tick === undefined)
return undefined;
out.push({ body: tick.body, kind: "command" });
i = tick.close;
continue;
}
if (ch === "$" && body[i + 1] === "(") {
const end = parenBodyEnd(body, i + 1);
if (typeof end !== "number")
return undefined;
out.push({ body: body.slice(i + 2, end - 1), kind: body[i + 2] === "(" ? "arithmetic" : "command" });
i = end - 1;
}
}
return out;
}
const BLANK = /[ \t]/;
const fail = (reason, at) => ({ parseFailure: { reason, at } });
const publicPiece = (p) => ({
span: [p.start, p.end],
quotedBy: p.quotedBy,
...(p.literal !== undefined ? { literal: p.literal } : {}),
...(p.expansion !== undefined ? { expansion: p.expansion } : {}),
});
class WordBuilder {
raw = "";
folded = "";
started = false;
at = 0;
pieces = [];
rawQuoted = [];
foldedAt = [];
srcOfRaw = [];
rawOfFolded = [];
marks = [];
subscript;
compound = false;
begin(src) { if (!this.started)
this.at = src; }
push(w, f, by, src) {
const rawStart = this.raw.length;
for (let k = 0; k < w.length; k++) {
this.rawQuoted.push(by);
this.foldedAt.push(this.folded.length);
this.srcOfRaw.push(src + k);
}
for (let k = 0; k < f.length; k++)
this.rawOfFolded.push(rawStart);
this.raw += w;
this.folded += f;
this.started = true;
}
emit(w, f, by, src, srcEnd) {
this.begin(src);
const last = this.pieces[this.pieces.length - 1];
if (last !== undefined && last.literal !== undefined && last.quotedBy === by && last.end === src && last.rawEnd === this.raw.length) {
last.literal += f;
last.end = srcEnd;
last.rawEnd += w.length;
last.foldedEnd += f.length;
}
else {
this.pieces.push({ start: src, end: srcEnd, rawStart: this.raw.length, rawEnd: this.raw.length + w.length, foldedStart: this.folded.length, foldedEnd: this.folded.length + f.length, quotedBy: by, literal: f, expansion: undefined });
}
this.push(w, f, by, src);
}
opaque(w, f, by, src, srcEnd, exp) {
this.begin(src);
this.pieces.push({ start: src, end: srcEnd, rawStart: this.raw.length, rawEnd: this.raw.length + w.length, foldedStart: this.folded.length, foldedEnd: this.folded.length + f.length, quotedBy: by, literal: undefined, expansion: exp });
this.push(w, f, by, src);
}
quoteChar(ch, by, src) {
this.begin(src);
this.rawQuoted.push(by);
this.foldedAt.push(this.folded.length);
this.srcOfRaw.push(src);
this.raw += ch;
this.started = true;
}
mark(reason, rawAt) { this.marks.push({ at: rawAt, reason }); }
inSubscript() { return this.subscript !== undefined && this.subscript.rawEnd === undefined; }
foldedOf(r) { return r >= this.foldedAt.length ? this.folded.length : this.foldedAt[r]; }
slice(from, to) {
const out = [];
for (const p of this.pieces) {
if (p.rawEnd <= from || p.rawStart >= to)
continue;
if (p.rawStart >= from && p.rawEnd <= to) {
out.push(p);
continue;
}
const rs = Math.max(p.rawStart, from);
const re = Math.min(p.rawEnd, to);
const fs = this.foldedOf(rs);
const fe = this.foldedOf(re);
out.push({
...p,
start: this.srcOfRaw[rs],
end: this.srcOfRaw[re - 1] + 1,
rawStart: rs,
rawEnd: re,
foldedStart: fs,
foldedEnd: fe,
literal: p.literal === undefined ? undefined : p.literal.slice(fs - p.foldedStart, fe - p.foldedStart),
});
}
return out;
}
expansionPiece(from, to, exp) {
return { start: this.srcOfRaw[from], end: this.srcOfRaw[to - 1] + 1, rawStart: from, rawEnd: to, foldedStart: this.foldedOf(from), foldedEnd: this.foldedOf(to), quotedBy: undefined, literal: undefined, expansion: exp };
}
carve(from, to, exp, inExp) {
this.pieces = [...this.slice(0, from), this.expansionPiece(from, to, exp), ...this.slice(to, this.raw.length)];
if (inExp !== undefined)
for (let k = from; k < to; k++)
inExp[k] = true;
}
carveAll(ranges) {
if (ranges.length === 0)
return;
const cut = (p, rs, re) => {
if (rs === p.rawStart && re === p.rawEnd)
return p;
const fs = this.foldedOf(rs);
const fe = this.foldedOf(re);
return { ...p, start: this.srcOfRaw[rs], end: this.srcOfRaw[re - 1] + 1, rawStart: rs, rawEnd: re, foldedStart: fs, foldedEnd: fe, literal: p.literal === undefined ? undefined : p.literal.slice(fs - p.foldedStart, fe - p.foldedStart) };
};
const out = [];
let r = 0;
for (const p of this.pieces) {
let cur = p.rawStart;
while (r < ranges.length && ranges[r].from < p.rawEnd) {
const rg = ranges[r];
if (rg.from >= cur) {
if (rg.from > cur)
out.push(cut(p, cur, rg.from));
out.push(this.expansionPiece(rg.from, rg.to, rg.exp));
}
if (rg.to >= p.rawEnd) {
cur = p.rawEnd;
break;
}
cur = rg.to;
r++;
}
if (cur < p.rawEnd)
out.push(cut(p, cur, p.rawEnd));
if (r < ranges.length && ranges[r].to === p.rawEnd)
r++;
}
this.pieces = out;
}
inExpansion() {
const out = new Array(this.raw.length).fill(false);
for (const p of this.pieces)
if (p.expansion !== undefined)
for (let k = p.rawStart; k < p.rawEnd; k++)
out[k] = true;
return out;
}
tildePrefixEnd(at, inValue) {
if (this.raw[at] !== "~" || this.rawQuoted[at] !== undefined)
return undefined;
let k = at + 1;
for (; k < this.raw.length; k++) {
if (this.rawQuoted[k] !== undefined)
return undefined;
const c = this.raw[k];
if (c === "/" || (inValue && c === ":"))
break;
}
return k;
}
carveTilde(from, to) {
const covering = this.pieces.find((p) => p.literal !== undefined && p.rawStart <= from && p.rawEnd >= to);
if (covering === undefined)
return;
this.carve(from, to, { kind: "tilde", yields: "one", body: this.raw.slice(from, to) });
}
carveBraceAndGlob() {
const inExp = this.inExpansion();
const n = this.raw.length;
const readable = (j) => this.rawQuoted[j] === undefined && !inExp[j];
const braceClose = new Map();
const stack = [];
for (let j = 0; j < n; j++) {
if (!readable(j))
continue;
const d = this.raw[j];
if (d === "{")
stack.push({ open: j, listed: false });
else if (d === "}") {
const top = stack.pop();
if (top !== undefined)
braceClose.set(top.open, { close: j, listed: top.listed });
}
else if (stack.length > 0 && (d === "," || (d === "." && this.raw[j + 1] === ".")))
stack[stack.length - 1].listed = true;
}
const nextBracketClose = new Array(n + 1).fill(-1);
for (let j = n - 1; j >= 0; j--)
nextBracketClose[j] = readable(j) && this.raw[j] === "]" ? j : nextBracketClose[j + 1];
const ranges = [];
for (let k = 0; k < n; k++) {
if (!readable(k))
continue;
const c = this.raw[k];
if (c === "{") {
const m = braceClose.get(k);
if (m === undefined || !m.listed)
continue;
ranges.push({ from: k, to: m.close + 1, exp: { kind: "brace", yields: "many", body: this.raw.slice(k + 1, m.close) } });
k = m.close;
continue;
}
if (c === "*" || c === "?") {
ranges.push({ from: k, to: k + 1, exp: { kind: "glob", yields: "many" } });
continue;
}
if (c === "[") {
let j = k + 1;
if (this.raw[j] === "!" || this.raw[j] === "^")
j++;
if (this.raw[j] === "]")
j++;
const close = j < n ? nextBracketClose[j] : -1;
if (close < 0)
continue;
ranges.push({ from: k, to: close + 1, exp: { kind: "glob", yields: "many", body: this.raw.slice(k + 1, close) } });
k = close;
}
}
this.carveAll(ranges);
}
readAssignment() {
const m = NAME_HEAD.exec(this.folded);
if (m === null)
return undefined;
const name = m[0];
const inExp = this.inExpansion();
const quotedF = (f) => { const r = this.rawOfFolded[f]; return r === undefined || this.rawQuoted[r] !== undefined; };
let at = name.length;
let sub;
if (this.folded[at] === "[") {
let depth = 0;
let close = -1;
for (let k = at; k < this.folded.length; k++) {
const r = this.rawOfFolded[k];
if (this.rawQuoted[r] !== undefined || inExp[r])
continue;
if (this.folded[k] === "[")
depth++;
else if (this.folded[k] === "]" && --depth === 0) {
close = k;
break;
}
}
if (close < 0)
return undefined;
sub = { from: at, to: close + 1 };
at = close + 1;
}
const appends = this.folded.startsWith("+=", at);
if (!appends && this.folded[at] !== "=")
return undefined;
const opLen = appends ? 2 : 1;
let bare = true;
for (let f = 0; f < name.length; f++)
if (quotedF(f))
bare = false;
if (sub !== undefined && (quotedF(sub.from) || quotedF(sub.to - 1)))
bare = false;
for (let f = at; f < at + opLen; f++)
if (quotedF(f))
bare = false;
const opRaw = this.rawOfFolded[at];
const valueRawFrom = opRaw + opLen;
if (!this.compound) {
for (let k = valueRawFrom; k < this.raw.length; k++) {
if (k !== valueRawFrom && !(this.raw[k - 1] === ":" && this.rawQuoted[k - 1] === undefined && !inExp[k - 1]))
continue;
const end = this.tildePrefixEnd(k, true);
if (end !== undefined)
this.carveTilde(k, end);
}
}
const valuePieces = this.slice(valueRawFrom, this.raw.length).map(publicPiece);
const value = {
raw: this.raw.slice(valueRawFrom),
folded: this.folded.slice(at + opLen),
pieces: valuePieces,
expansions: valuePieces.filter((p) => p.expansion !== undefined),
...(this.compound ? { unresolved: "compound_array_value" } : {}),
};
let subscript;
if (sub !== undefined) {
const rawFrom = this.rawOfFolded[sub.from] + 1;
const rawTo = this.rawOfFolded[sub.to - 1];
const inner = this.slice(rawFrom, rawTo).map(publicPiece);
subscript = { raw: this.raw.slice(rawFrom, rawTo), folded: this.folded.slice(sub.from + 1, sub.to - 1), expansions: inner.filter((p) => p.expansion !== undefined) };
}
return {
name,
op: appends ? "+=" : "=",
...(subscript !== undefined ? { subscript } : {}),
value,
bare,
form: this.compound ? "compound" : "scalar",
settlesWholeName: !appends && sub === undefined && !this.compound,
};
}
isAssignmentHead() {
if (this.compound || this.inSubscript())
return false;
if (!/^[A-Za-z_][A-Za-z0-9_]*(?:\[[\s\S]*\])?\+?=$/.test(this.raw))
return false;
const sub = this.subscript;
for (let k = 0; k < this.raw.length; k++) {
if (sub !== undefined && k > sub.rawStart && k < sub.rawEnd - 1)
continue;
if (this.rawQuoted[k] !== undefined)
return false;
}
return true;
}
finish(atCommandPosition, tildeSpelling = true) {
const rawText = this.raw;
let foldedText = this.folded;
let assignment;
const tildeEnd = this.tildePrefixEnd(0, false);
if (tildeEnd !== undefined)
this.carveTilde(0, tildeEnd);
else {
if (tildeSpelling)
foldedText = literalTildeSpelling(foldedText);
assignment = this.readAssignment();
}
if (!(assignment !== undefined && assignment.bare && atCommandPosition))
this.carveBraceAndGlob();
if (assignment !== undefined && this.subscript?.text)
this.mark("subscript_text", this.subscript.rawStart);
const marks = this.marks.filter((m) => m.reason !== "subscript_text" || assignment?.subscript !== undefined).sort((a, b) => a.at - b.at);
const unresolved = marks[0]?.reason ?? (assignment?.value.unresolved);
const spans = [];
for (let i = 0; i < this.rawQuoted.length; i++) {
const by = this.rawQuoted[i];
if (by === undefined)
continue;
const last = spans[spans.length - 1];
if (last !== undefined && last.by === by && last.end === i)
spans[spans.length - 1] = { ...last, end: i + 1 };
else
spans.push({ start: i, end: i + 1, by });
}
const pieces = this.pieces.map(publicPiece);
return {
kind: "word",
raw: rawText,
folded: foldedText,
pieces,
quotedSpans: spans,
expansions: pieces.filter((p) => p.expansion !== undefined),
...(assignment !== undefined ? { assignment } : {}),
...(unresolved !== undefined ? { unresolved } : {}),
atCommandPosition,
};
}
}
function scan(source, connectorsAreText) {
const split = !connectorsAreText;
const segments = [{ start: 0, end: source.length, connector: "start", tokens: [], substitutions: [] }];
const current = () => segments[segments.length - 1];
const pendingHeredocs = [];
let pendingRedirect;
let pendingHeredoc;
let inSingle = false;
let inDouble = false;
let regionBy;
let regionAt = 0;
let pendingLocale = false;
let acceptable = true;
let timeGrammar;
let definitionName = false;
let afterLeadingRedirect = false;
let assignmentSeen = false;
let subscriptsClosed = false;
let commandWord;
let w = new WordBuilder();
const next = (j) => past(source, j + 1);
const quotedBy = () => (inSingle || inDouble ? regionBy : undefined);
const take = (ch, src) => w.emit(ch, ch, quotedBy(), src, src + 1);
const substitution = (from, to, body, kind) => {
const by = quotedBy();
w.opaque(source.slice(from, to), SUBSTITUTION_PLACEHOLDER, by, from, to, { kind, yields: kind === "command" && by === undefined ? "many" : "one", body });
current().substitutions.push(body);
};
const resetPosition = () => { acceptable = true; commandWord = undefined; afterLeadingRedirect = false; assignmentSeen = false; subscriptsClosed = false; timeGrammar = undefined; definitionName = false; };
const endWord = () => {
if (!w.started)
return undefined;
if (w.inSubscript())
return fail("unterminated_subscript", w.srcOfRaw[w.subscript.rawStart]);
const word = w.finish(acceptable, pendingHeredoc === undefined);
const at = w.at;
w = new WordBuilder();
if (pendingHeredoc !== undefined) {
const header = pendingHeredoc;
pendingHeredoc = undefined;
if (word.folded === "" || /[$`]/.test(word.raw))
return fail("heredoc_delimiter_expands", at);
const token = { kind: "heredoc", delimiter: word.folded, quoted: /["'\\]/.test(word.raw), body: "" };
current().tokens.push(token);
pendingHeredocs.push({ delimiter: word.folded, quoted: token.quoted, dash: header.dash, token });
if (acceptable) {
if (assignmentSeen)
subscriptsClosed = true;
else
afterLeadingRedirect = true;
}
return undefined;
}
if (pendingRedirect !== undefined) {
const { op, fd } = pendingRedirect;
pendingRedirect = undefined;
const target = { ...word, atCommandPosition: false };
const written = fd !== undefined ? { fd } : {};
if ((op === ">&" || op === "<&") && word.raw === word.folded && /^(?:\d+-?|-)$/.test(word.folded))
current().tokens.push({ kind: "redirect", op, ...written, dup: word.folded });
else
current().tokens.push({ kind: "redirect", op: op === ">&" ? "&>" : op === "<&" ? "<" : op, ...written, target });
if (acceptable) {
if (assignmentSeen)
subscriptsClosed = true;
else
afterLeadingRedirect = true;
}
return undefined;
}
current().tokens.push(word);
const bare = word.raw === word.folded;
if (definitionName) {
resetPosition();
return undefined;
}
if (acceptable && bare && word.folded === "function") {
acceptable = false;
definitionName = true;
return undefined;
}
if (acceptable && bare && (SHELL_CONTROL_PREFIXES.has(word.folded) || GROUP_WORDS.has(word.folded))) {
timeGrammar = word.folded === "time" ? "opt" : undefined;
return undefined;
}
if (acceptable && bare && timeGrammar === "opt" && word.folded === "-p") {
timeGrammar = "ign";
return undefined;
}
if (acceptable && bare && timeGrammar !== undefined && word.folded === "--") {
timeGrammar = undefined;
return undefined;
}
timeGrammar = undefined;
if (word.assignment !== undefined && word.assignment.bare && acceptable) {
assignmentSeen = true;
return undefined;
}
if (acceptable)
commandWord = bare ? word.folded : undefined;
acceptable = false;
return undefined;
};
const operandOwed = (at) => pendingRedirect !== undefined ? fail("redirect_without_target", at) : pendingHeredoc !== undefined ? fail("heredoc_without_delimiter", at) : undefined;
const closeSegment = (connector, at, from, pipesStderr = false) => {
const refusal = endWord() ?? operandOwed(at);
if (refusal !== undefined)
return refusal;
current().end = at;
segments.push({ start: from, end: source.length, connector, ...(pipesStderr ? { pipesStderr: true } : {}), tokens: [], substitutions: [] });
resetPosition();
return undefined;
};
const readHeredocBodies = (from) => {
let p = from;
for (const h of pendingHeredocs) {
const lines = [];
for (;;) {
if (p > source.length)
return fail("unterminated_heredoc", from);
let nl = source.indexOf("\n", p);
let line = nl < 0 ? source.slice(p) : source.slice(p, nl);
if (!h.quoted) {
while (nl >= 0 && /(?<!\\)(?:\\\\)*\\$/.test(line)) {
const after = source.indexOf("\n", nl + 1);
line = line.slice(0, -1) + (after < 0 ? source.slice(nl + 1) : source.slice(nl + 1, after));
nl = after;
}
}
if ((h.dash ? line.replace(/^\t+/, "") : line) === h.delimiter) {
p = nl < 0 ? source.length + 1 : nl + 1;
break;
}
if (nl < 0)
return fail("unterminated_heredoc", from);
lines.push(line);
p = nl + 1;
}
h.token.body = lines.join("\n");
}
pendingHeredocs.length = 0;
return Math.min(p, source.length);
};
for (let i = 0; i < source.length; i++) {
if (!inSingle) {
const j = past(source, i);
if (j >= source.length)
break;
i = j;
}
const ch = source[i];
if (ch === "$" && !inSingle && !inDouble && dollarOpensQuote(source, i)) {
const q = next(i);
if (source[q] === '"') {
w.quoteChar("$", '$"', i);
pendingLocale = true;
i = q - 1;
continue;
}
if (source[q] === "'") {
const close = ansiCQuoteClose(source, q);
if (close === undefined)
return fail("unterminated_ansi_c", i);
const decoded = decodeAnsiC(source.slice(q + 1, close));
const written = source.slice(i, close + 1);
if (decoded.unresolved) {
w.mark("ansi_c_escape", w.raw.length);
w.opaque(written, decoded.text, "$'", i, close + 1, undefined);
}
else
w.emit(written, decoded.text, "$'", i, close + 1);
if (w.inSubscript() && decoded.text.includes("]"))
w.subscript.text = true;
i = close;
continue;
}
}
if (ch === "'" && !inDouble) {
if (inSingle) {
w.quoteChar(ch, "'", i);
inSingle = false;
regionBy = undefined;
}
else {
w.quoteChar(ch, "'", i);
inSingle = true;
regionBy = "'";
regionAt = i;
}
continue;
}
if (ch === '"' && !inSingle) {
if (inDouble) {
w.quoteChar(ch, regionBy ?? '"', i);
inDouble = false;
regionBy = undefined;
}
else {
const by = pendingLocale ? '$"' : '"';
pendingLocale = false;
if (by === '$"')
w.mark("locale_string", w.raw.length - 1);
w.quoteChar(ch, by, i);
inDouble = true;
regionBy = by;
regionAt = i;
}
continue;
}
if (inSingle) {
if (w.inSubscript() && ch === "]")
w.subscript.text = true;
take(ch, i);
continue;
}
if (!inDouble && ch === "[" && w.started && w.subscript === undefined && !w.compound && acceptable && !subscriptsClosed && IDENTIFIER.test(w.raw)) {
w.subscript = { rawStart: w.raw.length, depth: 1, text: false };
take(ch, i);
continue;
}
if (w.inSubscript() && !inDouble) {
const sub = w.subscript;
if (ch === "[") {
sub.depth++;
take(ch, i);
continue;
}
if (ch === "]") {
take(ch, i);
if (--sub.depth === 0)
sub.rawEnd = w.raw.length;
continue;
}
if (/[\n;|&<>]/.test(ch)) {
sub.text = true;
take(ch, i);
continue;
}
if (BLANK.test(ch)) {
if (afterLeadingRedirect)
sub.text = true;
take(ch, i);
continue;
}
if (ch === "\\" && source[i + 1] === "]")
sub.text = true;
}
if (w.inSubscript() && inDouble && ch === "]")
w.subscript.text = true;
const n1 = next(i);
if (ch === "\\") {
const escaped = source[i + 1];
if (escaped === undefined) {
take(ch, i);
continue;
}
if (inDouble && !"$`\"\\".includes(escaped))
w.emit(ch + escaped, ch + escaped, quotedBy(), i, i + 2);
else
w.emit(ch + escaped, escaped, inDouble ? quotedBy() : "\\", i, i + 2);
i++;
continue;
}
if (ch === "`") {
const tick = backtickBody(source, i);
if (tick === undefined)
return fail("unterminated_command_substitution", i);
substitution(i, tick.close + 1, tick.body, "command");
i = tick.close;
continue;
}
if (ch === "$" && source[n1] === "(") {
const arithmetic = source[next(n1)] === "(";
const end = parenBodyEnd(source, n1);
if (end === undefined)
return fail("unterminated_command_substitution", i);
if (end === "unreadable")
return fail("undelimited_substitution", i);
substitution(i, end, source.slice(n1 + 1, end - 1), arithmetic ? "arithmetic" : "command");
i = end - 1;
continue;
}
if (ch === "$" && source[n1] === "{") {
const end = braceExpansionEnd(source, n1);
if (end === undefined)
return fail("unterminated_parameter_expansion", i);
const body = source.slice(n1 + 1, end - 1);
const text = source.slice(i, end);
const name = EXPANSION_NAME.exec(body)?.[1];
if (UNREADABLE_EXPANSION_TEXT.test(body))
w.mark("parameter_text", w.raw.length);
w.opaque(text, text, quotedBy(), i, end, { kind: "parameter", yields: inDouble ? "one" : "many", body, ...(name !== undefined ? { name } : {}) });
i = end - 1;
continue;
}
if (ch === "$") {
const head = BARE_PARAMETER.exec(source.slice(n1, n1 + 64))?.[0];
if (head !== undefined) {
const text = source.slice(i, n1 + head.length);
w.opaque(text, text, quotedBy(), i, n1 + head.length, { kind: "parameter", yields: inDouble ? "one" : "many", name: head });
i = n1 + head.length - 1;
continue;
}
}
if (inDouble) {
take(ch, i);
continue;
}
if ((ch === "<" || ch === ">") && source[n1] === "(") {
const end = parenBodyEnd(source, n1);
if (end === undefined)
return fail("unterminated_process_substitution", i);
if (end === "unreadable")
return fail("undelimited_substitution", i);
substitution(i, end, source.slice(n1 + 1, end - 1), "process");
i = end - 1;
continue;
}
if (ch === "(" && w.started && w.isAssignmentHead() && (acceptable || (commandWord !== undefined && COMPOUND_OPERAND_COMMANDS.has(commandWord)))) {
const end = matchedParen(source, i);
if (end === undefined)
return fail("unterminated_compound_value", i);
const inner = scan(source.slice(i + 1, end - 1), true);
if ("parseFailure" in inner)
return fail(inner.parseFailure.reason, i + 1 + inner.parseFailure.at);
const text = source.slice(i, end);
w.mark("compound_array_value", w.raw.length);
w.opaque(text, text, undefined, i, end, undefined);
w.compound = true;
current().substitutions.push(...inner.command.substitutions);
i = end - 1;
continue;
}
if (w.inSubscript()) {
take(ch, i);
continue;
}
if (ch === "#" && !w.started) {
const nl = source.indexOf("\n", i);
i = nl < 0 ? source.length : nl - 1;
continue;
}
if (ch === "\n") {
if (!split) {
const r = endWord();
if (r !== undefined)
return r;
continue;
}
const refusal = closeSegment("\n", i, i + 1);
if (refusal !== undefined)
return refusal;
if (pendingHeredocs.length > 0) {
const resume = readHeredocBodies(i + 1);
if (typeof resume !== "number")
return resume;
current().start = resume;
i = resume - 1;
}
continue;
}
if (ch === "(" || ch === ")") {
const r = endWord() ?? operandOwed(i);
if (r !== undefined)
return r;
current().tokens.push({ kind: "group", op: ch });
resetPosition();
continue;
}
if (ch === "<" || ch === ">" || (ch === "&" && source[n1] === ">")) {
let fd;
if (ch !== "&" && w.started && w.raw === w.folded && /^\d+$/.test(w.raw)) {
fd = Number(w.raw);
w = new WordBuilder();
}
else {
const r = endWord() ?? operandOwed(i);
if (r !== undefined)
return r;
}
const written = fd !== undefined ? { fd } : {};
const n2 = next(n1);
if (ch === "&") {
if (source[n2] === ">") {
pendingRedirect = { op: "&>>" };
i = n2;
}
else {
pendingRedirect = { op: "&>" };
i = n1;
}
continue;
}
if (ch === "<" && source[n1] === "<") {
if (source[n2] === "<") {
pendingRedirect = { op: "<<<", ...written };
i = n2;
continue;
}
const dash = source[n2] === "-";
pendingHeredoc = { dash };
i = dash ? n2 : n1;
continue;
}
if (source[n1] === "&") {
pendingRedirect = { op: ch === ">" ? ">&" : "<&", ...written };
i = n1;
continue;
}
if (ch === ">" && (source[n1] === ">" || source[n1] === "|")) {
pendingRedirect = { op: source[n1] === ">" ? ">>" : ">", ...written };
i = n1;
continue;
}
pendingRedirect = { op: ch, ...written };
continue;
}
if (split) {
let refusal;
if (ch === ";")
refusal = closeSegment(";", i, i + 1);
else if (ch === "&") {
if (source[n1] === "&") {
refusal = closeSegment("&&", i, n1 + 1);
i = n1;
}
else
refusal = closeSegment("&", i, i + 1);
}
else if (ch === "|") {
if (source[n1] === "|" || source[n1] === "&") {
refusal = closeSegment(source[n1] === "|" ? "||" : "|", i, n1 + 1, source[n1] === "&");
i = n1;
}
else
refusal = closeSegment("|", i, i + 1);
}
else if (BLANK.test(ch)) {
refusal = endWord();
}
else {
take(ch, i);
continue;
}
if (refusal !== undefined)
return refusal;
continue;
}
if (BLANK.test(ch)) {
const r = endWord();
if (r !== undefined)
return r;
continue;
}
take(ch, i);
}
if (inSingle)
return fail("unterminated_single_quote", regionAt);
if (inDouble)
return fail("unterminated_double_quote", regionAt);
const tail = endWord() ?? operandOwed(source.length);
if (tail !== undefined)
return tail;
if (pendingHeredocs.length > 0)
return fail("unterminated_heredoc", source.length);
const out = segments.map((s) => ({
text: source.slice(s.start, s.end),
connector: s.connector,
...(s.pipesStderr ? { pipesStderr: true } : {}),
tokens: s.tokens,
substitutions: s.substitutions,
}));
return { command: { segments: out, substitutions: out.flatMap((s) => s.substitutions) } };
}
export function lexShell(source) {
const pass = scan(source, false);
return "parseFailure" in pass ? pass : pass.command;
}
export function lexShellText(source) {
const lexed = lexShell(source);
if ("parseFailure" in lexed)
return { reject: PARSE_FAILURE_SENTENCE[lexed.parseFailure.reason] };
for (const segment of lexed.segments) {
for (const token of segment.tokens) {
const word = token.kind === "word" ? token : token.kind === "redirect" ? token.target : undefined;
if (word?.unresolved !== undefined)
return { reject: UNRESOLVED_SENTENCE[word.unresolved] };
}
}
return lexed;
}
export function bashOnlyConstructsOf(command) {
const found = new Set();
const word = (w) => {
for (const s of w.quotedSpans) {
if (s.by === "$'")
found.add("ansi_c_quoting");
if (s.by === '$"')
found.add("locale_string");
}
for (const p of w.expansions) {
if (p.expansion?.kind === "process")
found.add("process_substitution");
if (p.expansion?.kind === "brace")
found.add("brace_expansion");
}
if (w.assignment !== undefined) {
if (w.assignment.op === "+=")
found.add("append_assignment");
if (w.assignment.subscript !== undefined)
found.add("subscript_assignment");
if (w.assignment.form === "compound")
found.add("compound_assignment");
}
};
for (const segment of command.segments) {
if (segment.pipesStderr)
found.add("pipe_both");
for (const t of segment.tokens) {
if (t.kind === "word")
word(t);
else if (t.kind === "redirect") {
if (t.op === "<<<")
found.add("here_string");
if (t.op === "&>" || t.op === "&>>")
found.add("redirect_both");
if (t.target !== undefined)
word(t.target);
}
}
}
return BASH_ONLY_CONSTRUCTS.filter((c) => found.has(c));
}