@fmlang/tokenizer/lib/main.js

Tokenizer for Forms Markup Language (FML)

// @bun
// src/Result.ts
class Result {
  value;
  error;
  constructor(value, error) {
    this.value = value;
    this.error = error;
  }
  IsError() {
    return this.error != null || this.value == null;
  }
  HasValue() {
    return this.value != null;
  }
}

// src/Token.ts
class Token {
  token_type;
  lexeme;
  literal;
  line;
  constructor(token_type, lexeme, literal, line) {
    this.token_type = token_type;
    this.lexeme = lexeme;
    this.literal = literal;
    this.line = line;
  }
}

// src/TokenType.ts
var TokenType = {
  IDENTIFIER: 0,
  STRING: 1,
  NUMBER: 2,
  MARKDOWN: 3,
  ASTERISK: 4,
  COMMA: 5,
  DOT: 6,
  MINUS: 7,
  PLUS: 8,
  SEMICOLON: 9,
  FORWARD_SLASH: 10,
  BACK_SLASH: 11,
  POUND: 12,
  LEFT_BRACE: 13,
  RIGHT_BRACE: 14,
  LEFT_BRACKET: 15,
  RIGHT_BRACKET: 16,
  LEFT_PARENTHESIS: 17,
  RIGHT_PARENTHESIS: 18,
  EXCLAMATION_POINT: 19,
  EXCLAMATION_POINT_EQUAL: 20,
  EQUAL: 21,
  EQUAL_EQUAL: 22,
  GREATER_THAN: 23,
  GREATER_THAN_EQUAL: 24,
  LESS_THAN: 25,
  LESS_THAN_EQUAL: 26,
  AND: 27,
  ELSE: 28,
  FALSE: 29,
  LOOP: 30,
  IF: 31,
  OR: 32,
  PRINT: 33,
  RETURN: 34,
  TRUE: 35,
  LET: 36,
  CONST: 37,
  EOL: 99,
  EOF: 100
};

// src/KeywordMap.ts
var KeywordMap = {
  and: TokenType.AND,
  else: TokenType.ELSE,
  false: TokenType.FALSE,
  loop: TokenType.LOOP,
  if: TokenType.IF,
  or: TokenType.OR,
  print: TokenType.PRINT,
  return: TokenType.RETURN,
  true: TokenType.TRUE,
  let: TokenType.LET,
  const: TokenType.CONST
};

// src/Tokenizer.ts
class Tokenizer {
  tokens = [];
  start = 0;
  current = 0;
  line = 1;
  source = "";
  constructor(fml) {
    this.source = fml;
    this.start = 0;
    this.tokens = [];
    this.current = 0;
    this.line = 1;
  }
  Tokenize() {
    while (!this.IsAtEnd()) {
      this.start = this.current;
      this.ScanToken();
    }
    this.tokens.push(new Token(TokenType.EOF, "", null, this.line));
    return new Result(this.tokens, null);
  }
  IsAtEnd() {
    return this.current >= this.source.length;
  }
  ScanToken() {
    let c = this.Advance();
    switch (c) {
      case "(":
        this.AddToken(TokenType.LEFT_PARENTHESIS);
        break;
      case ")":
        this.AddToken(TokenType.RIGHT_PARENTHESIS);
        break;
      case "{":
        this.AddToken(TokenType.LEFT_BRACE);
        break;
      case "}":
        this.AddToken(TokenType.RIGHT_BRACE);
        break;
      case "[":
        this.AddToken(TokenType.LEFT_BRACKET);
        break;
      case "]":
        this.AddToken(TokenType.RIGHT_BRACKET);
        break;
      case ",":
        this.AddToken(TokenType.COMMA);
        break;
      case ".":
        this.AddToken(TokenType.DOT);
        break;
      case "-":
        this.AddToken(TokenType.MINUS);
        break;
      case "+":
        this.AddToken(TokenType.PLUS);
        break;
      case "*":
        this.AddToken(TokenType.ASTERISK);
        break;
      case "":
        this.AddToken(TokenType.SEMICOLON);
        break;
      case "!":
        this.AddToken(this.MatchNext("=") ? TokenType.EXCLAMATION_POINT_EQUAL : TokenType.EXCLAMATION_POINT);
        break;
      case "=":
        this.AddToken(this.MatchNext("=") ? TokenType.EQUAL_EQUAL : TokenType.EQUAL);
        break;
      case "<":
        this.AddToken(this.MatchNext("=") ? TokenType.LESS_THAN_EQUAL : TokenType.LESS_THAN);
        break;
      case ">":
        this.AddToken(this.MatchNext("=") ? TokenType.GREATER_THAN_EQUAL : TokenType.GREATER_THAN);
        break;
      case "/": {
        if (this.MatchNext("/")) {
          while (this.PeekNext() != `
` && !this.IsAtEnd()) {
            this.Advance();
          }
        } else {
          this.AddToken(TokenType.FORWARD_SLASH);
        }
        break;
      }
      case "#": {
        if (this.MatchNext("{")) {
          this.ReadMarkdown();
        }
        this.AddToken(TokenType.POUND);
        break;
      }
      case '"': {
        this.ReadString();
        break;
      }
      case " ":
      case "\r":
      case "\t":
        break;
      case `
`:
        this.line++;
        break;
      default:
        if (this.IsDigit(c)) {
          this.ReadNumber();
        } else if (this.IsAlpha(c)) {
          this.ReadIdentifier();
        } else {
          throw new Error(`Unexpected character ${c}`);
        }
        break;
    }
  }
  Advance() {
    this.current++;
    return this.source[this.current - 1];
  }
  MatchNext(expected) {
    if (this.IsAtEnd())
      return false;
    if (this.source[this.current] != expected)
      return false;
    this.current++;
    return true;
  }
  IsDigit(lexeme) {
    return lexeme >= "0" && lexeme <= "9";
  }
  ReadNumber() {
    while (this.IsDigit(this.PeekNext())) {
      this.Advance();
    }
    if (this.PeekNext() == "." && this.IsDigit(this.PeekNext(1))) {
      this.Advance();
      while (this.IsDigit(this.PeekNext())) {
        this.Advance();
      }
    }
    let num = parseFloat(this.source.substring(this.start, this.current));
    this.AddToken(TokenType.NUMBER, num);
  }
  PeekNext(offset = 0) {
    if (this.IsAtEnd())
      return "\x00";
    return this.source[this.current + offset];
  }
  ReadString() {
    while (this.PeekNext() != '"' && !this.IsAtEnd()) {
      let next = this.PeekNext();
      if (this.PeekNext() == "\\" && this.PeekNext(1) == '"') {
        this.Advance();
      }
      if (this.PeekNext() == `
`) {
        this.line++;
      }
      this.Advance();
    }
    if (this.IsAtEnd()) {
      throw new Error("Unterminated string on line " + this.line);
    }
    this.Advance();
    let value = this.source.substring(this.start + 1, this.current - 1);
    value = value.replace(/\\"/g, '"');
    this.AddToken(TokenType.STRING, value);
  }
  IsAlpha(c) {
    return c >= "a" && c <= "z" || c >= "A" && c <= "Z" || c == "_";
  }
  IsAlphaNumeric(c) {
    return this.IsAlpha(c) || this.IsDigit(c);
  }
  ReadIdentifier() {
    while (this.IsAlphaNumeric(this.PeekNext())) {
      this.Advance();
    }
    let text = this.source.substring(this.start, this.current);
    let token_type = KeywordMap[text] || TokenType.IDENTIFIER;
    this.AddToken(token_type);
  }
  AddToken(token_type, literal = null) {
    let text = this.source.substring(this.start, this.current);
    this.tokens.push(new Token(token_type, text, literal, this.line));
  }
  ReadMarkdown() {
    while (!this.IsAtEnd()) {
      if (this.PeekNext() != "}") {
        this.Advance();
      } else {
        if (this.PeekNext(1) == "#") {
          break;
        } else {
          if (this.IsAtEnd()) {
            throw new Error(`Unterminated markdown on line ${this.line}: ${this.source.substring(this.start)}`);
          }
          this.Advance();
        }
      }
    }
    if (this.IsAtEnd()) {
      throw new Error(`Unterminated markdown on line ${this.line}: ${this.source.substring(this.start)}`);
    }
    if (!this.MatchNext("}")) {
      throw new Error(`Unterminated markdown on line ${this.line}: ${this.source.substring(this.start)}`);
    }
    if (!this.MatchNext("#")) {
      throw new Error(`Unterminated markdown on line ${this.line}: ${this.source.substring(this.start)}`);
    }
    let value = this.source.substring(this.start + 2, this.current - 2);
    value = value.trim().split(`
`).map((line) => line.trim()).join(`
`);
    this.AddToken(TokenType.MARKDOWN, value);
    this.line += value.split(`
`).length + 1;
    this.start = this.current;
  }
}
export {
  Tokenizer,
  TokenType,
  Token,
  Result,
  KeywordMap
};