kestrel.markets/src/tooling/highlight.ts

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # highlight — a THIN highlight projection over the lexer (src/lang/lex.ts)
 *
 * Syntax highlighting for Kestrel, built as a **projection** over the real lexer rather than
 * a second lexer (ADR-0004: text is a projection of one object model, and the lexer is the
 * inverse projection's first stage). {@link tokenize} calls {@link lex} — the same tokenizer
 * the parser reads — and maps each lexeme (word / number / string / punct) to a themeable
 * highlight category, filling the whitespace and comment gaps the lexer drops so the token
 * stream **tiles** the source (no gaps, no overlaps). A renderer can therefore reconstruct
 * the exact text from the tokens; {@link highlightToHtml} and {@link highlightToAnsi} are the
 * two zero-dependency renderers (web + terminal) that do.
 *
 * Portability (ARCHITECTURE — "portability is the point"): this module imports ONLY
 * `src/lang` and pure TS. No DOM, no `node:*`, no new deps — so the same core drives the
 * VS Code extension, a Next.js web view, and a terminal.
 *
 * Fail closed: {@link tokenize} never throws. A line the lexer rejects (an unterminated
 * string, a tab indent, a stray character) degrades to a `text` token for that line's content
 * so the rest of the buffer still highlights — highlighting an in-progress buffer must not
 * crash the editor.
 */

import { lex, type Token } from "../lang/lex.ts";

/**
 * A themeable highlight category. `word` lexemes split into {@link classifyWord}'s three
 * cases (keyword / anchor / ident); `punct` splits into operator vs punctuation. `whitespace`
 * and `comment` are reconstructed from the gaps the lexer drops; `text` is the fail-closed
 * catch-all for a line the lexer could not tokenize. Duration/series are grammar-level (a
 * number+unit, a dotted path) rather than lexical, so this lexical projection does not coin
 * them — a renderer that wants them re-derives them from the parse, not from these tokens.
 */
export type HighlightKind =
  | "keyword"
  | "anchor"
  | "operator"
  | "punctuation"
  | "number"
  | "string"
  | "comment"
  | "ident"
  | "whitespace"
  | "text";

/** One highlight span over the source. `[start, end)` are 0-based character offsets into the
 * original text; the tokens returned by {@link tokenize} tile `[0, text.length)` exactly. */
export interface HighlightToken {
  readonly start: number;
  readonly end: number;
  readonly kind: HighlightKind;
}

// ─────────────────────────────────────────────────────────────────────────────
// Vocabulary — kept in step with src/lang (parse.ts + print.ts). Case-sensitive,
// matching the grammar (canonical keywords are case-sensitive).
// ─────────────────────────────────────────────────────────────────────────────

/** Price anchors (ADR-0005: anchors are the only legitimate price handles). */
const ANCHORS: ReadonlySet<string> = new Set([
  "fair",
  "intrinsic",
  "basis",
  "bid",
  "ask",
  "mid",
  "last",
  "join",
  "improve",
  "stub",
]);

/** Every reserved word the grammar dispatches on — statement + module directives, clause
 * heads, trigger/price/leg operators, and the enum-like literals the parser matches by name.
 * A word not in here (and not an anchor) is an open identifier (a series/plan/pane name). */
const KEYWORDS: ReadonlySet<string> = new Set([
  // statement + module directives
  "VIEW", "WAKE", "PLAN", "GRADE", "POD", "BOOK", "IMPORT", "PROVENANCE", "USING", "FROM",
  // clause heads
  "WHEN", "DO", "ALSO", "RELOAD", "TP", "EXIT", "INVALIDATE", "CANCEL-IF", "ARM", "BECAUSE",
  "DELIVER", "PRIORITY", "COALESCE", "BUDGET", "UNIVERSE", "VS", "BY", "OVER", "FILL", "RISK",
  "MANDATORY", "KEYFRAME",
  // trigger algebra
  "AND", "OR", "NOT", "held", "within", "until", "at", "crosses", "touches", "above", "below", "of", "band",
  "phase", "time", "after", "before", "leg",
  // fill events
  "filled", "partial-fill", "unfilled", "rejected", "cancelled",
  // ordinals
  "zeroth", "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth",
  "ninth", "tenth", "eleventh", "twelfth",
  // plan/book header attributes
  "budget", "ttl", "regime", "priority", "standing", "prov", "atomic",
  "signal", "exec", "coverage", "thesis", "arbitration", "concurrency",
  // order policy
  "peg", "fix", "esc", "cap", "floor", "cancel-if", "gtc",
  // legs, quantifiers, prices
  "buy", "sell", "atm", "min", "max", "lean", "foreach", "any", "all", "frac",
  "C", "P",
  // grade counterfactuals + dimensions
  "ungated", "null", "bracket", "vehicle", "name", "lineage",
  // misc literals
  "sigma", "dte", "wakes", "tokens", "day",
]);

/** Puncts that read as operators (relational / structural), vs. mere grouping punctuation. */
const OPERATOR_PUNCT: ReadonlySet<string> = new Set([
  ">", "<", ">=", "<=", "==", "!=", "->", "..", "+", "-", "%", "/", "@",
]);

function classifyWord(text: string): HighlightKind {
  if (ANCHORS.has(text)) return "anchor";
  if (KEYWORDS.has(text)) return "keyword";
  return "ident";
}

function classify(t: Token): HighlightKind {
  switch (t.type) {
    case "number":
      return "number";
    case "string":
      return "string";
    case "word":
      return classifyWord(t.text);
    case "punct":
      return OPERATOR_PUNCT.has(t.text) ? "operator" : "punctuation";
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// Tiling — reconstruct the whitespace + comment the lexer drops so tokens tile
// ─────────────────────────────────────────────────────────────────────────────

/** Fill one intra-line gap `[start, end)` (no newline inside) with a whitespace token and, if
 * a `#` opens a comment, a comment token running to the end of the line. A `#` can only appear
 * in a gap outside every lexeme (strings are captured as tokens spanning their quotes), so any
 * `#` here is a genuine comment. */
function fillGap(src: string, start: number, end: number, out: HighlightToken[]): void {
  const hash = src.indexOf("#", start);
  if (hash !== -1 && hash < end) {
    if (hash > start) out.push({ start, end: hash, kind: "whitespace" });
    out.push({ start: hash, end, kind: "comment" });
  } else {
    out.push({ start, end, kind: "whitespace" });
  }
}

/** Tile a single physical line `[base, base+line.length)` by running the real lexer over it
 * and filling the gaps. On a lexer escape, degrade the line to leading-whitespace + a `text`
 * token (fail closed — the rest of the document still highlights).
 *
 * Canonical Kestrel is LF (.editorconfig), but a CRLF buffer leaves a trailing `\r` on each
 * line after the `\n` split. The lexer rejects `\r` as a stray character, so we peel it off as
 * a trailing whitespace token and lex only the body — the `\r` still tiles, and CRLF buffers
 * highlight as cleanly as LF ones instead of degrading every line to a fail-closed `text`. */
function tileLine(src: string, base: number, line: string, out: HighlightToken[]): void {
  const hasCr = line.endsWith("\r");
  const body = hasCr ? line.slice(0, -1) : line;
  const bodyEnd = base + body.length;
  const toks: Token[] = [];
  try {
    for (const root of lex(body).roots) collectTokens(root, toks);
  } catch {
    const ws = /^\s*/.exec(body)![0].length;
    if (ws > 0) out.push({ start: base, end: base + ws, kind: "whitespace" });
    if (body.length > ws) out.push({ start: base + ws, end: bodyEnd, kind: "text" });
    if (hasCr) out.push({ start: bodyEnd, end: bodyEnd + 1, kind: "whitespace" });
    return;
  }
  const proj = toks
    .map((t) => ({ start: base + (t.col - 1), end: base + (t.end - 1), kind: classify(t) }))
    .sort((a, b) => a.start - b.start);
  let cursor = base;
  for (const t of proj) {
    if (t.start > cursor) fillGap(src, cursor, t.start, out);
    out.push(t);
    cursor = t.end;
  }
  if (cursor < bodyEnd) fillGap(src, cursor, bodyEnd, out);
  if (hasCr) out.push({ start: bodyEnd, end: bodyEnd + 1, kind: "whitespace" });
}

/** A lexer {@link LineNode} carries its own tokens plus deeper continuation lines; a single
 * physical line has no children, but we recurse defensively so this stays correct if `lex`
 * ever nests. */
function collectTokens(node: { tokens: readonly Token[]; children: readonly unknown[] }, out: Token[]): void {
  out.push(...node.tokens);
  for (const child of node.children) collectTokens(child as never, out);
}

/**
 * Project the source into a tiling stream of {@link HighlightToken}s by running the real lexer
 * (line by line) and reconstructing the whitespace/comment gaps it drops. The tokens cover
 * `[0, text.length)` with no gaps or overlaps, so `text.slice(t.start, t.end)` concatenated
 * over the stream reproduces the source exactly. Never throws.
 */
export function tokenize(text: string): HighlightToken[] {
  const out: HighlightToken[] = [];
  const lines = text.split("\n");
  let base = 0;
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]!;
    tileLine(text, base, line, out);
    base += line.length;
    if (i < lines.length - 1) {
      out.push({ start: base, end: base + 1, kind: "whitespace" });
      base += 1;
    }
  }
  return out;
}

// ─────────────────────────────────────────────────────────────────────────────
// Pure renderers (zero-dep) — web (HTML) + terminal (ANSI)
// ─────────────────────────────────────────────────────────────────────────────

function escapeHtml(s: string): string {
  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
}

/**
 * Render `text` as themeable HTML: each non-whitespace token becomes
 * `<span class="kestrel-<kind>">…</span>` (HTML-escaped); whitespace is emitted raw. No DOM
 * API is touched, so this runs anywhere (server-render, worker, edge). Stripping the spans and
 * un-escaping the entities reproduces the source exactly.
 */
export function highlightToHtml(text: string): string {
  let out = "";
  for (const t of tokenize(text)) {
    const raw = escapeHtml(text.slice(t.start, t.end));
    out += t.kind === "whitespace" ? raw : `<span class="kestrel-${t.kind}">${raw}</span>`;
  }
  return out;
}

/** SGR color code per highlight category; a `kind` absent here (ident, whitespace) renders in
 * the terminal's default color. */
const ANSI_CODE: Partial<Record<HighlightKind, number>> = {
  keyword: 35, // magenta
  anchor: 36, // cyan
  operator: 33, // yellow
  punctuation: 90, // bright black (dim)
  number: 34, // blue
  string: 32, // green
  comment: 90, // dim
  text: 91, // bright red — a lexer escape, surfaced loudly
};

/**
 * Render `text` for a terminal with ANSI SGR colors. Pure token → string; stripping the SGR
 * escapes (`\x1b[…m`) reproduces the source exactly.
 */
export function highlightToAnsi(text: string): string {
  let out = "";
  for (const t of tokenize(text)) {
    const raw = text.slice(t.start, t.end);
    const code = ANSI_CODE[t.kind];
    out += code === undefined ? raw : `\x1b[${code}m${raw}\x1b[0m`;
  }
  return out;
}