kestrel.markets/dist/lang/lex.js

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # lex — the indentation-aware lexer (text -> a line tree of tokens)
 *
 * The inverse projection of the printer begins here (ADR-0004). Kestrel is line-oriented:
 * a statement **header** sits at its own indentation and its **clauses** sit one level
 * deeper (ADR-0005: "line continuation is solved by an indentation-aware lexer from day
 * one"). This lexer does two jobs:
 *
 * 1. **Tokenize** each physical line into {@link Token}s that carry `line`/`col` so every
 *    parse error can name exactly where it happened (fail-closed, ARCHITECTURE §6).
 * 2. **Nest** physical lines into a {@link LineNode} tree by indentation, so the parser
 *    can tell a statement's clauses (deeper) from its siblings (same indent) — and so a
 *    **continued clause line** (deeper still) folds into the clause above it.
 *
 * The tokenizer is deliberately small: WORD / NUMBER / STRING / PUNCT. The tight lexical
 * forms Kestrel prints — `fair-3c`, `velocity(1m)`, `p99`, `0dte`, `zoom-1d` — are
 * reassembled by the *parser* from adjacent tokens (it knows the grammar position), not
 * guessed at by the lexer.
 */
/** A parse (or lex) failure. Carries `line`/`col` and an instructive message — the text is
 * read by LLM agents, so it names what was expected and what was found (ARCHITECTURE §6:
 * fail-closed, never degrade silently). */
export class KestrelParseError extends Error {
    line;
    col;
    name = "KestrelParseError";
    constructor(message, line, col) {
        super(`${message} (line ${line}, col ${col})`);
        this.line = line;
        this.col = col;
    }
}
// Multi-character puncts must be tried before their single-character prefixes.
const PUNCT2 = ["..", "->", ">=", "<=", "==", "!="];
const PUNCT1 = [">", "<", "+", "-", "%", ":", ",", ".", "(", ")", "{", "}", "@", "/"];
function isDigit(ch) {
    return ch >= "0" && ch <= "9";
}
function isWordStart(ch) {
    return (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || ch === "_";
}
function isWordPart(ch) {
    return isWordStart(ch) || isDigit(ch);
}
/**
 * Tokenize one physical line. `content` is the whole line text; scanning starts at the
 * first non-space column. A `#` outside a string ends the line (comment). A hyphen is
 * absorbed into a WORD only when the following character is a word character, so
 * `failed-break` / `cancel-if` stay single tokens while `fair-3c` splits into
 * `fair` `-` `3` `c` (the parser reassembles offsets and hyphen-digit names by adjacency).
 *
 * A `#` outside a string opens a TRAILING-INLINE comment (ADR-0033): scanning stops and the
 * verbatim text after `#` (to end of line, `#` excluded) is returned as `trailing` so the
 * printer can re-emit it byte-identically instead of dropping it.
 */
function tokenizeLine(content, lineNo) {
    const toks = [];
    let i = 0;
    const n = content.length;
    while (i < n) {
        const ch = content[i];
        if (ch === " " || ch === "\t") {
            i++;
            continue;
        }
        if (ch === "#")
            return { tokens: toks, trailing: content.slice(i + 1) }; // comment to end of line
        const col = i + 1;
        if (ch === '"') {
            // String literal: decode escapes; unterminated is fail-closed.
            let j = i + 1;
            let out = "";
            while (j < n) {
                const c = content[j];
                if (c === "\\") {
                    const nx = content[j + 1];
                    if (nx === undefined) {
                        throw new KestrelParseError("unterminated string escape", lineNo, j + 1);
                    }
                    out += nx;
                    j += 2;
                    continue;
                }
                if (c === '"')
                    break;
                out += c;
                j++;
            }
            if (j >= n || content[j] !== '"') {
                throw new KestrelParseError('unterminated string: expected a closing "', lineNo, col);
            }
            j++; // consume closing quote
            toks.push({ type: "string", text: out, line: lineNo, col, end: j + 1 });
            i = j;
            continue;
        }
        if (isDigit(ch)) {
            let j = i + 1;
            while (j < n && isDigit(content[j]))
                j++;
            // fractional part only if a digit follows the dot
            if (content[j] === "." && j + 1 < n && isDigit(content[j + 1])) {
                j += 2;
                while (j < n && isDigit(content[j]))
                    j++;
            }
            toks.push({ type: "number", text: content.slice(i, j), line: lineNo, col, end: j + 1 });
            i = j;
            continue;
        }
        if (isWordStart(ch)) {
            let j = i + 1;
            while (j < n) {
                const c = content[j];
                if (isWordPart(c)) {
                    j++;
                    continue;
                }
                // absorb a hyphen only when it glues two word parts (failed-break), not fair-3c
                if (c === "-" && j + 1 < n && isWordStart(content[j + 1])) {
                    j += 2;
                    continue;
                }
                break;
            }
            toks.push({ type: "word", text: content.slice(i, j), line: lineNo, col, end: j + 1 });
            i = j;
            continue;
        }
        // punctuation (2-char first)
        const two = content.slice(i, i + 2);
        if (PUNCT2.includes(two)) {
            toks.push({ type: "punct", text: two, line: lineNo, col, end: i + 3 });
            i += 2;
            continue;
        }
        if (PUNCT1.includes(ch)) {
            toks.push({ type: "punct", text: ch, line: lineNo, col, end: i + 2 });
            i++;
            continue;
        }
        throw new KestrelParseError(`unexpected character ${JSON.stringify(ch)}`, lineNo, col);
    }
    return { tokens: toks };
}
/** Count leading spaces (indentation). A tab is fail-closed — canonical Kestrel is
 * 2-space (`.editorconfig`); mixing tabs would make indentation ambiguous. */
function leadingIndent(content, lineNo) {
    let i = 0;
    while (i < content.length && content[i] === " ")
        i++;
    if (content[i] === "\t") {
        throw new KestrelParseError("tab in indentation: Kestrel indents with spaces", lineNo, i + 1);
    }
    return i;
}
/**
 * Lex a whole document into a forest of top-level {@link LineNode}s. Blank lines are dropped
 * (the canonical printer owns vertical spacing — ADR-0033). Comment lines are NOT dropped:
 * an own-line `#` comment is captured as the `leading` trivia of the next code line at its
 * indentation, or — when it closes a block (it is more deeply indented than the line that
 * follows, or it ends the document) — as the `tail` of the block it belongs to. An inline
 * `#…` comment rides its code line as `trailing`. Each code line nests under the nearest
 * preceding line with strictly smaller indentation, so the tree encodes both block structure
 * (a statement and its clauses) and line continuation (a deeper line under a clause).
 */
export function lex(src) {
    const physical = src.split("\n");
    const roots = [];
    const stack = [];
    let pending = [];
    const moduleTail = [];
    /** Attach a block-closing own-line comment as the `tail` of the deepest enclosing block on
     * the stack (the deepest node whose indent is strictly smaller), or to the module tail. */
    const attachTail = (c) => {
        for (let s = stack.length - 1; s >= 0; s--) {
            const node = stack[s];
            if (node.indent < c.indent) {
                node.tail = [...(node.tail ?? []), c.text];
                return;
            }
        }
        moduleTail.push(c.text);
    };
    for (let p = 0; p < physical.length; p++) {
        const content = physical[p];
        const lineNo = p + 1;
        const trimmed = content.trim();
        if (trimmed === "")
            continue; // blank line — normalized away
        if (trimmed.startsWith("#")) {
            // Own-line comment: capture verbatim text after `#`, with its indentation, and hold it
            // until the next code line (leading) or a dedent/EOF (tail) resolves where it belongs.
            const indent = leadingIndent(content, lineNo);
            pending.push({ indent, text: content.slice(content.indexOf("#") + 1) });
            continue;
        }
        const indent = leadingIndent(content, lineNo);
        const { tokens, trailing } = tokenizeLine(content, lineNo);
        if (tokens.length === 0)
            continue; // defensive: a line with no tokens and no leading `#`
        // Pending comments more indented than this line close deeper block(s) → their tails; the
        // rest lead this line. Resolve tails BEFORE popping so the closing blocks are still on the
        // stack.
        const leading = [];
        for (const c of pending) {
            if (c.indent > indent)
                attachTail(c);
            else
                leading.push(c.text);
        }
        pending = [];
        const node = { indent, line: lineNo, col: tokens[0].col, tokens, children: [] };
        if (leading.length > 0)
            node.leading = leading;
        if (trailing !== undefined)
            node.trailing = trailing;
        while (stack.length > 0 && stack[stack.length - 1].indent >= indent)
            stack.pop();
        if (stack.length === 0)
            roots.push(node);
        else
            stack[stack.length - 1].children.push(node);
        stack.push(node);
    }
    // Trailing own-line comments after the last code line: tails of their enclosing block, or module.
    for (const c of pending)
        attachTail(c);
    return { roots, tail: moduleTail };
}
/** Flatten a line and all its continuation descendants into a single token stream. Used
 * for leaf lines (a statement header, a clause, a RISK line) where deeper-indented lines
 * are continuations rather than nested statements. */
export function flatten(node) {
    const out = [...node.tokens];
    for (const child of node.children)
        out.push(...flatten(child));
    return out;
}