@sema-agent/core/dist/core/shell-lexer.js

Stateless, task-oriented AI agent core

import { shellWordsOf } from "../shell/shell-words.js";
import { evaluatesOperand, SHELL_WRAPPER_TABLE } from "./shell-wrapper-table.js";
import { arithmeticHidesRun, carriesSubstitution, freshWord, matchOp, readDoubleQuoted, readExpansion, skipBalanced, skipHeredocBodies } from "./shell-scan.js";
export { SHELL_WRAPPER_TABLE } from "./shell-wrapper-table.js";
const WRAPPERS = new Map(Object.entries(SHELL_WRAPPER_TABLE));
const PREFIX_KEYWORDS = shellWordsOf("stripped_prefix");
const CLOSING_KEYWORDS = shellWordsOf("stripped_standalone");
const OPAQUE_KEYWORDS = shellWordsOf("opaque");
const ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*(?:\[[^\]]*\])?\+?=/;
const DURATION_RE = /^\d+(?:\.\d+)?[smhd]?$/;
const REDIRECT_OPS = ["<<<", "<<-", "<<", "<>", "<&", "<", ">>", ">|", ">&", "&>>", "&>", ">"];
const CONNECTOR_OPS = ["&&", "||", "|&", "|", ";;&", ";&", ";;", ";", "&"];
const ALL_OPS = [...CONNECTOR_OPS, ...REDIRECT_OPS];
export const MAX_SHELL_READ_CHARS = 10_000;
export function readShellCommand(source) {
    if (source.length > MAX_SHELL_READ_CHARS)
        return { segments: [{ argv: [], redirections: [], peeled: [], unreadable: `the command is longer than ${MAX_SHELL_READ_CHARS} characters — not read` }], connectors: [], grouped: false, backgrounded: false, strayRedirection: false };
    const segments = [];
    const connectors = [];
    let grouped = false;
    let backgrounded = false;
    let strayRedirection = false;
    let seg = { words: [], redirections: [] };
    let word = freshWord();
    let pendingRedirect;
    let heredocs = [];
    let badDelimiter = false;
    let afterBoundary = true;
    let lastConnector;
    let tail;
    const n = source.length;
    let i = 0;
    const endWord = () => {
        if (!word.started)
            return;
        if (word.braceExpands)
            word.expands = "many";
        if (word.substitution)
            seg.substitution = true;
        if (pendingRedirect !== undefined) {
            if (pendingRedirect === "<<" || pendingRedirect === "<<-") {
                if (word.expands !== false)
                    badDelimiter = true;
                heredocs.push({ delimiter: word.text, quoted: word.raw !== word.text, dash: pendingRedirect === "<<-" });
            }
            seg.redirections.push({ op: pendingRedirect, target: publicWord(word) });
            pendingRedirect = undefined;
        }
        else {
            seg.words.push(word);
        }
        word = freshWord();
    };
    const finishSegment = (connector) => {
        endWord();
        if (pendingRedirect !== undefined) {
            seg.unreadable ??= `the redirection \`${pendingRedirect}\` has no operand — a syntax error`;
            pendingRedirect = undefined;
        }
        let built = buildSegment(seg);
        if (built === "keyworded")
            grouped = true;
        if ((built === "empty" || built === "keyworded") && seg.redirections.length > 0)
            strayRedirection = true;
        if (built === "empty" || built === "keyworded") {
            const continuation = connector === "\n" && lastConnector !== undefined && lastConnector !== ";" && lastConnector !== "&" && lastConnector !== "\n";
            const legal = built === "keyworded" || afterBoundary || connector === undefined || connector === "\n" || lastConnector === "\n";
            if (!legal)
                built = { argv: [], redirections: seg.redirections, peeled: [], unreadable: `an empty command before \`${connector}\` — a syntax error` };
            else {
                if (connector !== undefined && connector !== "\n" && segments.length > 0)
                    grouped = true;
                if (connector !== undefined && !continuation)
                    lastConnector = connector;
                seg = { words: [], redirections: [] };
                afterBoundary = false;
                return;
            }
        }
        if (segments.length > 0)
            connectors.push(lastConnector ?? "\n");
        segments.push(built);
        if (connector !== undefined)
            lastConnector = connector;
        seg = { words: [], redirections: [] };
        afterBoundary = false;
    };
    const boundary = () => { finishSegment(undefined); grouped = true; afterBoundary = true; lastConnector = undefined; };
    while (i < n && tail === undefined) {
        const ch = source[i];
        if (ch === " " || ch === "\t") {
            endWord();
            i++;
            continue;
        }
        if (ch === "\n") {
            endWord();
            if (badDelimiter) {
                tail = "a here-document delimiter spelled with an expansion";
                break;
            }
            if (heredocs.length === 0) {
                finishSegment("\n");
                i++;
                continue;
            }
            const skipped = skipHeredocBodies(source, i + 1, heredocs);
            heredocs = [];
            if (skipped === undefined) {
                tail = "a here-document body never reaches its delimiter";
                break;
            }
            if (skipped.substitution)
                seg.substitution = true;
            finishSegment("\n");
            i = skipped.end;
            continue;
        }
        if (ch === "#" && !word.started) {
            while (i < n && source[i] !== "\n")
                i++;
            continue;
        }
        if (ch === "\\") {
            if (i + 1 >= n) {
                tail = "a trailing backslash — the line never ends";
                break;
            }
            if (source[i + 1] === "\n") {
                i += 2;
                continue;
            }
            word.started = true;
            word.text += source[i + 1];
            word.raw += source.slice(i, i + 2);
            i += 2;
            continue;
        }
        if (ch === "'") {
            const close = source.indexOf("'", i + 1);
            if (close < 0) {
                tail = "an unterminated single quote";
                break;
            }
            word.started = true;
            word.text += source.slice(i + 1, close);
            word.raw += source.slice(i, close + 1);
            i = close + 1;
            continue;
        }
        let dq = i + 1;
        if (ch === "$")
            while (source[dq] === "\\" && source[dq + 1] === "\n")
                dq += 2;
        if (ch === '"' || (ch === "$" && source[dq] === '"')) {
            const open = ch === "$" ? dq + 1 : i + 1;
            const end = readDoubleQuoted(source, open, word);
            if (end === undefined) {
                tail = "an unterminated double quote";
                break;
            }
            word.raw += source.slice(i, end);
            i = end;
            continue;
        }
        if (ch === "$" || ch === "`" || ((ch === "<" || ch === ">") && source[i + 1] === "(")) {
            const end = readExpansion(source, i, word);
            if (end === undefined) {
                tail = "an unterminated substitution";
                break;
            }
            if (end === i + 1) {
                word.started = true;
                word.text += ch;
                word.raw += ch;
                i = end;
                continue;
            }
            word.started = true;
            word.expands = ch === "$" || ch === "`" ? "many" : word.expands || "one";
            word.raw += source.slice(i, end);
            i = end;
            continue;
        }
        if (ch === "~" && !word.started)
            word.expands = "one";
        const op = matchOp(source, i, ALL_OPS);
        if (op !== undefined) {
            if (CONNECTOR_OPS.includes(op)) {
                if (op === ";;" || op === ";&" || op === ";;&") {
                    endWord();
                    seg.unreadable ??= "a `case` arm terminator outside a `case` this lane reads";
                    i += op.length;
                    continue;
                }
                if (op === "&")
                    backgrounded = true;
                finishSegment(op);
                i += op.length;
                continue;
            }
            if (word.started && word.raw === word.text && (/^\d+$/.test(word.text) || /^\{[A-Za-z_][A-Za-z0-9_]*\}$/.test(word.text)))
                word = freshWord();
            else
                endWord();
            if (pendingRedirect !== undefined)
                seg.unreadable ??= `the redirection \`${pendingRedirect}\` has no operand — a syntax error`;
            pendingRedirect = op;
            i += op.length;
            continue;
        }
        if (ch === "(" && word.started && "@?*+!".includes(word.raw.slice(-1)) && word.raw !== "!") {
            const end = skipBalanced(source, i, "(", ")");
            if (end === undefined) {
                tail = "an unterminated extended pattern (`@(…)`)";
                break;
            }
            if (carriesSubstitution(source.slice(i + 1, end - 1)))
                word.substitution = true;
            word.expands = "many";
            word.text += source.slice(i, end);
            word.raw += source.slice(i, end);
            i = end;
            continue;
        }
        if (ch === "(" && source[i + 1] === "(" && (!word.started || word.raw === "!") && pendingRedirect === undefined && seg.words.every((w) => w.expands === false && (shellWordsOf("control_prefix").has(w.text) || CLOSING_KEYWORDS.has(w.text) || w.text === "-p"))) {
            endWord();
            const end = skipBalanced(source, i, "(", ")");
            if (end === undefined) {
                tail = "an unterminated arithmetic command";
                break;
            }
            if (arithmeticHidesRun(source.slice(i + 2, end - 1)))
                seg.substitution = true;
            i = end;
            continue;
        }
        if (ch === "(" || ch === ")") {
            if (ch === "(" && word.started && !word.expands && seg.words.length === 0 && pendingRedirect === undefined) {
                let j = i + 1;
                while (j < n && (source[j] === " " || source[j] === "\t"))
                    j++;
                if (source[j] === ")") {
                    word = freshWord();
                    boundary();
                    i = j + 1;
                    continue;
                }
            }
            boundary();
            i++;
            continue;
        }
        if (ch === "*" || ch === "?" || (ch === "]" && word.bracketOpen))
            word.expands = "many";
        if (ch === "[")
            word.bracketOpen = true;
        if (ch === "{")
            word.braceOpen = true;
        if (word.braceOpen && (ch === "," || (ch === "." && source[i + 1] === ".")))
            word.braceExpands = true;
        if (ch === "}" && !word.braceExpands)
            word.braceOpen = false;
        word.started = true;
        word.text += ch;
        word.raw += ch;
        i++;
    }
    if (tail !== undefined) {
        endWord();
        seg.unreadable ??= tail;
        finishSegment(undefined);
    }
    else {
        if (heredocs.length > 0)
            seg.unreadable ??= "a here-document with no body";
        const dangling = seg.words.length === 0 && !word.started && lastConnector !== undefined && lastConnector !== ";" && lastConnector !== "&" && lastConnector !== "\n";
        if (dangling)
            seg.unreadable ??= `the connector \`${lastConnector}\` has no command after it — a syntax error`;
        finishSegment(undefined);
    }
    return { segments, connectors, grouped, backgrounded, strayRedirection };
}
const publicWord = ({ text, raw, expands }) => ({ text, raw, expands });
const SUBSTITUTION_UNREAD = "a command substitution runs a program this reader does not read";
const MAX_WRAPPER_LAYERS = 8;
export function isFullyReadable(shape) {
    return shape.segments.every((s) => s.unreadable === undefined && s.peelUnreadable === undefined && [s.argv, ...s.peeled].every((argv) => argv[0]?.expands === false));
}
function buildSegment(seg) {
    if (seg.unreadable !== undefined)
        return { argv: seg.words.map(publicWord), redirections: seg.redirections, peeled: [], unreadable: seg.unreadable };
    const carried = () => (seg.substitution === true ? { argv: [], redirections: seg.redirections, peeled: [], peelUnreadable: SUBSTITUTION_UNREAD } : keyworded ? "keyworded" : "empty");
    let keyworded = false;
    let k0 = 0;
    while (k0 < seg.words.length && seg.words[k0].expands === false && (PREFIX_KEYWORDS.has(seg.words[k0].text) || CLOSING_KEYWORDS.has(seg.words[k0].text))) {
        k0++;
        keyworded = true;
    }
    const words = k0 === 0 ? seg.words : seg.words.slice(k0);
    const head = words[0];
    if (head === undefined)
        return carried();
    if (evaluatesOperand(words))
        seg.substitution = true;
    if (head.expands === false && (head.text === "[[" || head.text === "["))
        return carried();
    if (head.expands === false && OPAQUE_KEYWORDS.has(head.text))
        return { argv: words.map(publicWord), redirections: seg.redirections, peeled: [], unreadable: `a \`${head.text}\` control structure — its commands need a syntax tree` };
    const argv = words.map(publicWord);
    const peeled = [];
    let peelUnreadable;
    let k = 0;
    while (k < argv.length && ASSIGNMENT_RE.test(argv[k].raw))
        k++;
    let run = argv.slice(k);
    if (k > 0 && run.length > 0)
        peeled.push(run);
    for (let layer = 0; run.length > 0; layer++) {
        let j = 0;
        while (j < run.length && ((run[j].text === "!" && run[j].expands === false) || ASSIGNMENT_RE.test(run[j].raw)))
            j++;
        if (j > 0) {
            run = run.slice(j);
            if (run.length === 0)
                break;
            peeled.push(run);
        }
        if (run[0].expands !== false) {
            peelUnreadable = "the program name is an expansion";
            break;
        }
        if (layer >= MAX_WRAPPER_LAYERS) {
            peelUnreadable = `more than ${MAX_WRAPPER_LAYERS} wrapper layers`;
            break;
        }
        const next = unwrap(run);
        if (typeof next === "string") {
            peelUnreadable = next;
            break;
        }
        if (next === run)
            break;
        run = next;
        peeled.push(run);
    }
    if (seg.substitution === true || [argv, ...peeled].some((c) => c[0]?.expands === false && evaluatesOperand(c)))
        peelUnreadable ??= SUBSTITUTION_UNREAD;
    return { argv, redirections: seg.redirections, peeled, ...(peelUnreadable !== undefined ? { peelUnreadable } : {}) };
}
function unwrap(words) {
    const head = words[0];
    const name = head.text.slice(head.text.lastIndexOf("/") + 1);
    const row = WRAPPERS.get(name);
    if (row === undefined)
        return words;
    if (row.viaShell === true)
        return `\`${name}\` evaluates its operands as shell input this reader does not read`;
    if (row.executes !== undefined) {
        const hit = words.slice(1).find((w) => w.expands !== false || row.executes.includes(w.text));
        return hit === undefined ? words : hit.expands !== false ? `an expansion where \`${name}\` reads its arguments` : `\`${name} ${hit.text}\` runs a command this reader does not read`;
    }
    const valueFlags = new Set(row.valueFlags);
    const flags = new Set(row.flags);
    const attached = new Set(row.attached ?? []);
    const commandString = new Set(row.commandString);
    let positionalTaken = row.positional === undefined;
    let durationTaken = row.duration !== true;
    let replaceToken;
    let i = 1;
    const unknown = (t) => `the \`${name}\` option ${t} is not one this reader knows`;
    const takeValue = (flag) => {
        const value = words[i + 1];
        if (name === "xargs" && flag === "-I" && value !== undefined) {
            replaceToken = value.text;
            if (value.expands !== false)
                return false;
        }
        i += value === undefined ? 1 : 2;
        return value === undefined || value.expands !== "many";
    };
    const operand = (w) => {
        if (w.expands !== false)
            return `an expansion where \`${name}\` reads its operands`;
        if (!durationTaken) {
            durationTaken = true;
            return DURATION_RE.test(w.text) ? true : `the \`${name}\` duration "${w.text}" cannot be read`;
        }
        if (!positionalTaken) {
            positionalTaken = true;
            return row.positional.test(w.text) ? true : `the \`${name}\` operand "${w.text}" cannot be read`;
        }
        return false;
    };
    while (i < words.length) {
        const w = words[i];
        if (w.expands !== false)
            return `an expansion where \`${name}\` reads its options`;
        const t = w.text;
        if (t === "--") {
            i++;
            while (i < words.length && (!durationTaken || !positionalTaken)) {
                const r = operand(words[i]);
                if (r !== true)
                    return r;
                i++;
            }
            while (row.assignments === true && i < words.length && words[i].expands === false && ASSIGNMENT_RE.test(words[i].text))
                i++;
            break;
        }
        if (t === "-" && row.bareDash === true) {
            i++;
            continue;
        }
        if (name === "nice" && /^-\d+$/.test(t)) {
            i++;
            continue;
        }
        if (t.startsWith("-") && t !== "-") {
            if (name === "command" && /^-[pvV]+$/.test(t) && /[vV]/.test(t))
                return words;
            if (t.startsWith("--")) {
                const eq = t.indexOf("=");
                const flag = eq > 0 ? t.slice(0, eq) : t;
                if (commandString.has(flag))
                    return `\`${name} ${flag}\` takes its command as one string`;
                if (eq < 0 && valueFlags.has(flag)) {
                    if (!takeValue(flag))
                        return `an expansion where \`${name}\` reads its options`;
                    continue;
                }
                if (!flags.has(flag) && !(eq > 0 && valueFlags.has(flag)))
                    return unknown(flag);
                if (name === "xargs" && flag === "--replace")
                    replaceToken = eq > 0 ? t.slice(eq + 1) : "{}";
                i++;
                continue;
            }
            let k = 1;
            for (; k < t.length; k++) {
                const letter = `-${t[k]}`;
                if (commandString.has(letter))
                    return `\`${name} ${letter}\` takes its command as one string`;
                if (valueFlags.has(letter)) {
                    if (name === "xargs" && letter === "-I" && k < t.length - 1)
                        replaceToken = t.slice(k + 1);
                    break;
                }
                if (attached.has(letter)) {
                    if (name === "xargs" && letter === "-i")
                        replaceToken = t.length > k + 1 ? t.slice(k + 1) : "{}";
                    k = t.length;
                    break;
                }
                if (!flags.has(letter))
                    return unknown(letter);
            }
            if (k === t.length - 1) {
                if (!takeValue(`-${t[k]}`))
                    return `an expansion where \`${name}\` reads its options`;
                continue;
            }
            i++;
            continue;
        }
        if (row.assignments === true && ASSIGNMENT_RE.test(t)) {
            i++;
            continue;
        }
        const r = operand(w);
        if (r === false)
            break;
        if (r !== true)
            return r;
        i++;
    }
    if (i >= words.length)
        return words;
    const rest = words.slice(i);
    if (name !== "xargs")
        return rest;
    if (replaceToken === undefined)
        return [...rest, { text: "", raw: "<xargs input>", expands: "many" }];
    return rest.map((w) => (w.text.includes(replaceToken) || w.raw.includes(replaceToken) ? { ...w, expands: "many" } : w));
}