kestrel.markets/src/lang/lex.ts

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # lex — the indentation-aware lexer (text -> a line tree of tokens)
 *
 * The inverse projection of the printer begins here (ADR-0004). Kestrel is line-oriented:
 * a statement **header** sits at its own indentation and its **clauses** sit one level
 * deeper (ADR-0005: "line continuation is solved by an indentation-aware lexer from day
 * one"). This lexer does two jobs:
 *
 * 1. **Tokenize** each physical line into {@link Token}s that carry `line`/`col` so every
 *    parse error can name exactly where it happened (fail-closed, ARCHITECTURE §6).
 * 2. **Nest** physical lines into a {@link LineNode} tree by indentation, so the parser
 *    can tell a statement's clauses (deeper) from its siblings (same indent) — and so a
 *    **continued clause line** (deeper still) folds into the clause above it.
 *
 * The tokenizer is deliberately small: WORD / NUMBER / STRING / PUNCT. The tight lexical
 * forms Kestrel prints — `fair-3c`, `velocity(1m)`, `p99`, `0dte`, `zoom-1d` — are
 * reassembled by the *parser* from adjacent tokens (it knows the grammar position), not
 * guessed at by the lexer.
 */

/** A parse (or lex) failure. Carries `line`/`col` and an instructive message — the text is
 * read by LLM agents, so it names what was expected and what was found (ARCHITECTURE §6:
 * fail-closed, never degrade silently). */
export class KestrelParseError extends Error {
  override readonly name = "KestrelParseError";
  constructor(
    message: string,
    readonly line: number,
    readonly col: number,
  ) {
    super(`${message} (line ${line}, col ${col})`);
  }
}

export type TokenType = "word" | "number" | "string" | "punct";

/** One lexical token. `col`/`end` are 1-based columns within `line`; adjacency (no
 * intervening space) is `next.col === prev.end`, which the parser uses to reassemble tight
 * forms like `fair-3c`. For a `string` token, `text` is the DECODED value (quotes and
 * escapes removed). */
export interface Token {
  readonly type: TokenType;
  readonly text: string;
  readonly line: number;
  readonly col: number;
  /** 1-based column one past the last character (exclusive). */
  readonly end: number;
}

/** One physical line's tokens, plus any deeper-indented lines nested beneath it (its
 * clauses, or — for a continued line — its continuation).
 *
 * Comment trivia (ADR-0033, byte-stable round-trip): `leading` are the own-line `#` comments
 * that immediately precede this line (verbatim, `#` excluded); `trailing` is the inline `#`
 * comment on this same line (verbatim, `#` excluded); `tail` are the own-line comments that
 * fall INSIDE this block after its last child, at the block's interior indentation. All are
 * absent on a comment-free line. */
export interface LineNode {
  readonly indent: number;
  readonly line: number;
  /** 1-based column of the first token. */
  readonly col: number;
  readonly tokens: readonly Token[];
  readonly children: LineNode[];
  leading?: readonly string[];
  trailing?: string;
  tail?: readonly string[];
}

/** An own-line comment captured with the indentation at which it was written — the indent
 * decides which block it belongs to (leading of the next line vs. tail of the block it closes). */
interface PendingComment {
  readonly indent: number;
  /** Verbatim text after the opening `#`, to end of line (the `#` excluded). */
  readonly text: string;
}

// Multi-character puncts must be tried before their single-character prefixes.
const PUNCT2: readonly string[] = ["..", "->", ">=", "<=", "==", "!="];
const PUNCT1: readonly string[] = [">", "<", "+", "-", "%", ":", ",", ".", "(", ")", "{", "}", "@", "/"];

function isDigit(ch: string): boolean {
  return ch >= "0" && ch <= "9";
}
function isWordStart(ch: string): boolean {
  return (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || ch === "_";
}
function isWordPart(ch: string): boolean {
  return isWordStart(ch) || isDigit(ch);
}

/**
 * Tokenize one physical line. `content` is the whole line text; scanning starts at the
 * first non-space column. A `#` outside a string ends the line (comment). A hyphen is
 * absorbed into a WORD only when the following character is a word character, so
 * `failed-break` / `cancel-if` stay single tokens while `fair-3c` splits into
 * `fair` `-` `3` `c` (the parser reassembles offsets and hyphen-digit names by adjacency).
 *
 * A `#` outside a string opens a TRAILING-INLINE comment (ADR-0033): scanning stops and the
 * verbatim text after `#` (to end of line, `#` excluded) is returned as `trailing` so the
 * printer can re-emit it byte-identically instead of dropping it.
 */
function tokenizeLine(content: string, lineNo: number): { tokens: Token[]; trailing?: string } {
  const toks: Token[] = [];
  let i = 0;
  const n = content.length;
  while (i < n) {
    const ch = content[i]!;
    if (ch === " " || ch === "\t") {
      i++;
      continue;
    }
    if (ch === "#") return { tokens: toks, trailing: content.slice(i + 1) }; // comment to end of line
    const col = i + 1;

    if (ch === '"') {
      // String literal: decode escapes; unterminated is fail-closed.
      let j = i + 1;
      let out = "";
      while (j < n) {
        const c = content[j]!;
        if (c === "\\") {
          const nx = content[j + 1];
          if (nx === undefined) {
            throw new KestrelParseError("unterminated string escape", lineNo, j + 1);
          }
          out += nx;
          j += 2;
          continue;
        }
        if (c === '"') break;
        out += c;
        j++;
      }
      if (j >= n || content[j] !== '"') {
        throw new KestrelParseError('unterminated string: expected a closing "', lineNo, col);
      }
      j++; // consume closing quote
      toks.push({ type: "string", text: out, line: lineNo, col, end: j + 1 });
      i = j;
      continue;
    }

    if (isDigit(ch)) {
      let j = i + 1;
      while (j < n && isDigit(content[j]!)) j++;
      // fractional part only if a digit follows the dot
      if (content[j] === "." && j + 1 < n && isDigit(content[j + 1]!)) {
        j += 2;
        while (j < n && isDigit(content[j]!)) j++;
      }
      toks.push({ type: "number", text: content.slice(i, j), line: lineNo, col, end: j + 1 });
      i = j;
      continue;
    }

    if (isWordStart(ch)) {
      let j = i + 1;
      while (j < n) {
        const c = content[j]!;
        if (isWordPart(c)) {
          j++;
          continue;
        }
        // absorb a hyphen only when it glues two word parts (failed-break), not fair-3c
        if (c === "-" && j + 1 < n && isWordStart(content[j + 1]!)) {
          j += 2;
          continue;
        }
        break;
      }
      toks.push({ type: "word", text: content.slice(i, j), line: lineNo, col, end: j + 1 });
      i = j;
      continue;
    }

    // punctuation (2-char first)
    const two = content.slice(i, i + 2);
    if (PUNCT2.includes(two)) {
      toks.push({ type: "punct", text: two, line: lineNo, col, end: i + 3 });
      i += 2;
      continue;
    }
    if (PUNCT1.includes(ch)) {
      toks.push({ type: "punct", text: ch, line: lineNo, col, end: i + 2 });
      i++;
      continue;
    }

    throw new KestrelParseError(`unexpected character ${JSON.stringify(ch)}`, lineNo, col);
  }
  return { tokens: toks };
}

/** Count leading spaces (indentation). A tab is fail-closed — canonical Kestrel is
 * 2-space (`.editorconfig`); mixing tabs would make indentation ambiguous. */
function leadingIndent(content: string, lineNo: number): number {
  let i = 0;
  while (i < content.length && content[i] === " ") i++;
  if (content[i] === "\t") {
    throw new KestrelParseError("tab in indentation: Kestrel indents with spaces", lineNo, i + 1);
  }
  return i;
}

/** The result of lexing: the top-level line forest plus the module-level tail comments
 * (own-line `#` comments that fall after the last statement at column 0, owned by no block). */
export interface LexResult {
  readonly roots: LineNode[];
  /** Verbatim module-tail comment texts (`#` excluded), in source order. */
  readonly tail: readonly string[];
}

/**
 * Lex a whole document into a forest of top-level {@link LineNode}s. Blank lines are dropped
 * (the canonical printer owns vertical spacing — ADR-0033). Comment lines are NOT dropped:
 * an own-line `#` comment is captured as the `leading` trivia of the next code line at its
 * indentation, or — when it closes a block (it is more deeply indented than the line that
 * follows, or it ends the document) — as the `tail` of the block it belongs to. An inline
 * `#…` comment rides its code line as `trailing`. Each code line nests under the nearest
 * preceding line with strictly smaller indentation, so the tree encodes both block structure
 * (a statement and its clauses) and line continuation (a deeper line under a clause).
 */
export function lex(src: string): LexResult {
  const physical = src.split("\n");
  const roots: LineNode[] = [];
  const stack: LineNode[] = [];
  let pending: PendingComment[] = [];
  const moduleTail: string[] = [];

  /** Attach a block-closing own-line comment as the `tail` of the deepest enclosing block on
   * the stack (the deepest node whose indent is strictly smaller), or to the module tail. */
  const attachTail = (c: PendingComment): void => {
    for (let s = stack.length - 1; s >= 0; s--) {
      const node = stack[s]!;
      if (node.indent < c.indent) {
        node.tail = [...(node.tail ?? []), c.text];
        return;
      }
    }
    moduleTail.push(c.text);
  };

  for (let p = 0; p < physical.length; p++) {
    const content = physical[p]!;
    const lineNo = p + 1;
    const trimmed = content.trim();
    if (trimmed === "") continue; // blank line — normalized away
    if (trimmed.startsWith("#")) {
      // Own-line comment: capture verbatim text after `#`, with its indentation, and hold it
      // until the next code line (leading) or a dedent/EOF (tail) resolves where it belongs.
      const indent = leadingIndent(content, lineNo);
      pending.push({ indent, text: content.slice(content.indexOf("#") + 1) });
      continue;
    }

    const indent = leadingIndent(content, lineNo);
    const { tokens, trailing } = tokenizeLine(content, lineNo);
    if (tokens.length === 0) continue; // defensive: a line with no tokens and no leading `#`

    // Pending comments more indented than this line close deeper block(s) → their tails; the
    // rest lead this line. Resolve tails BEFORE popping so the closing blocks are still on the
    // stack.
    const leading: string[] = [];
    for (const c of pending) {
      if (c.indent > indent) attachTail(c);
      else leading.push(c.text);
    }
    pending = [];

    const node: LineNode = { indent, line: lineNo, col: tokens[0]!.col, tokens, children: [] };
    if (leading.length > 0) node.leading = leading;
    if (trailing !== undefined) node.trailing = trailing;

    while (stack.length > 0 && stack[stack.length - 1]!.indent >= indent) stack.pop();
    if (stack.length === 0) roots.push(node);
    else stack[stack.length - 1]!.children.push(node);
    stack.push(node);
  }

  // Trailing own-line comments after the last code line: tails of their enclosing block, or module.
  for (const c of pending) attachTail(c);

  return { roots, tail: moduleTail };
}

/** Flatten a line and all its continuation descendants into a single token stream. Used
 * for leaf lines (a statement header, a clause, a RISK line) where deeper-indented lines
 * are continuations rather than nested statements. */
export function flatten(node: LineNode): Token[] {
  const out: Token[] = [...node.tokens];
  for (const child of node.children) out.push(...flatten(child));
  return out;
}