zignet
Version:
MCP server for Zig — AI-powered code analysis, validation, and documentation with fine-tuned LLM
261 lines (259 loc) • 8.87 kB
JavaScript
//#region src/lexer.ts
/**
* ZigNet Lexer
* Tokenizes Zig source code
*/
let TokenType = /* @__PURE__ */ function(TokenType$1) {
TokenType$1["FN"] = "FN";
TokenType$1["CONST"] = "CONST";
TokenType$1["VAR"] = "VAR";
TokenType$1["STRUCT"] = "STRUCT";
TokenType$1["UNION"] = "UNION";
TokenType$1["ENUM"] = "ENUM";
TokenType$1["IF"] = "IF";
TokenType$1["ELSE"] = "ELSE";
TokenType$1["WHILE"] = "WHILE";
TokenType$1["FOR"] = "FOR";
TokenType$1["RETURN"] = "RETURN";
TokenType$1["BREAK"] = "BREAK";
TokenType$1["CONTINUE"] = "CONTINUE";
TokenType$1["COMPTIME"] = "COMPTIME";
TokenType$1["INLINE"] = "INLINE";
TokenType$1["I32"] = "I32";
TokenType$1["I64"] = "I64";
TokenType$1["U32"] = "U32";
TokenType$1["F32"] = "F32";
TokenType$1["F64"] = "F64";
TokenType$1["BOOL"] = "BOOL";
TokenType$1["VOID"] = "VOID";
TokenType$1["NUMBER"] = "NUMBER";
TokenType$1["STRING"] = "STRING";
TokenType$1["IDENT"] = "IDENT";
TokenType$1["TRUE"] = "TRUE";
TokenType$1["FALSE"] = "FALSE";
TokenType$1["PLUS"] = "PLUS";
TokenType$1["MINUS"] = "MINUS";
TokenType$1["STAR"] = "STAR";
TokenType$1["SLASH"] = "SLASH";
TokenType$1["PERCENT"] = "PERCENT";
TokenType$1["EQ"] = "EQ";
TokenType$1["NEQ"] = "NEQ";
TokenType$1["LT"] = "LT";
TokenType$1["GT"] = "GT";
TokenType$1["LTE"] = "LTE";
TokenType$1["GTE"] = "GTE";
TokenType$1["ASSIGN"] = "ASSIGN";
TokenType$1["PLUS_ASSIGN"] = "PLUS_ASSIGN";
TokenType$1["AND"] = "AND";
TokenType$1["OR"] = "OR";
TokenType$1["NOT"] = "NOT";
TokenType$1["LPAREN"] = "LPAREN";
TokenType$1["RPAREN"] = "RPAREN";
TokenType$1["LBRACE"] = "LBRACE";
TokenType$1["RBRACE"] = "RBRACE";
TokenType$1["LBRACKET"] = "LBRACKET";
TokenType$1["RBRACKET"] = "RBRACKET";
TokenType$1["COLON"] = "COLON";
TokenType$1["SEMICOLON"] = "SEMICOLON";
TokenType$1["COMMA"] = "COMMA";
TokenType$1["DOT"] = "DOT";
TokenType$1["ARROW"] = "ARROW";
TokenType$1["FAT_ARROW"] = "FAT_ARROW";
TokenType$1["EOF"] = "EOF";
TokenType$1["ERROR"] = "ERROR";
return TokenType$1;
}({});
var Token = class {
constructor(type, value, line, column) {
this.type = type;
this.value = value;
this.line = line;
this.column = column;
}
toString() {
return `Token(${this.type}, "${this.value}", ${this.line}:${this.column})`;
}
};
const KEYWORDS = {
fn: TokenType.FN,
const: TokenType.CONST,
var: TokenType.VAR,
struct: TokenType.STRUCT,
union: TokenType.UNION,
enum: TokenType.ENUM,
if: TokenType.IF,
else: TokenType.ELSE,
while: TokenType.WHILE,
for: TokenType.FOR,
return: TokenType.RETURN,
break: TokenType.BREAK,
continue: TokenType.CONTINUE,
comptime: TokenType.COMPTIME,
inline: TokenType.INLINE,
i32: TokenType.I32,
i64: TokenType.I64,
u32: TokenType.U32,
f32: TokenType.F32,
f64: TokenType.F64,
bool: TokenType.BOOL,
void: TokenType.VOID,
true: TokenType.TRUE,
false: TokenType.FALSE
};
const ESCAPE_SEQUENCES = {
n: "\n",
t: " ",
r: "\r",
"\\": "\\",
"\"": "\"",
"'": "'"
};
var Lexer = class {
position = 0;
line = 1;
column = 1;
tokens = [];
constructor(source) {
this.source = source;
}
error(message) {
return new Token(TokenType.ERROR, message, this.line, this.column);
}
peek(offset = 0) {
const pos = this.position + offset;
if (pos >= this.source.length) return "\0";
return this.source[pos];
}
advance() {
const char = this.source[this.position];
this.position++;
if (char === "\n") {
this.line++;
this.column = 1;
} else this.column++;
return char;
}
skipWhitespace() {
while (this.position < this.source.length && /\s/.test(this.peek())) this.advance();
}
skipComment() {
if (this.peek() === "/" && this.peek(1) === "/") {
while (this.peek() !== "\n" && this.peek() !== "\0") this.advance();
return true;
}
return false;
}
readIdentifier() {
let value = "";
while (/[a-zA-Z0-9_]/.test(this.peek())) value += this.advance();
return value;
}
readNumber() {
let value = "";
while (/[0-9]/.test(this.peek())) value += this.advance();
if (this.peek() === "." && /[0-9]/.test(this.peek(1))) {
value += this.advance();
while (/[0-9]/.test(this.peek())) value += this.advance();
}
return value;
}
readString(quote) {
let value = "";
this.advance();
while (this.peek() !== quote && this.peek() !== "\0") if (this.peek() === "\\") {
this.advance();
const escaped = this.advance();
value += this.getEscapeSequence(escaped);
} else value += this.advance();
if (this.peek() === quote) this.advance();
else return this.error("Unterminated string");
return value;
}
getEscapeSequence(char) {
return ESCAPE_SEQUENCES[char] || char;
}
tokenize() {
while (this.position < this.source.length) {
this.skipWhitespace();
if (this.skipComment()) continue;
const char = this.peek();
const line = this.line;
const column = this.column;
if (char === "\0") break;
if (/[0-9]/.test(char)) {
const value = this.readNumber();
this.tokens.push(new Token(TokenType.NUMBER, value, line, column));
continue;
}
if (char === "\"" || char === "'") {
const value = this.readString(char);
if (value instanceof Token) this.tokens.push(value);
else this.tokens.push(new Token(TokenType.STRING, value, line, column));
continue;
}
if (/[a-zA-Z_]/.test(char)) {
const value = this.readIdentifier();
const type = KEYWORDS[value] || TokenType.IDENT;
this.tokens.push(new Token(type, value, line, column));
continue;
}
this.advance();
if (char === "+") if (this.peek() === "=") {
this.advance();
this.tokens.push(new Token(TokenType.PLUS_ASSIGN, "+=", line, column));
} else this.tokens.push(new Token(TokenType.PLUS, "+", line, column));
else if (char === "-") if (this.peek() === ">") {
this.advance();
this.tokens.push(new Token(TokenType.ARROW, "->", line, column));
} else this.tokens.push(new Token(TokenType.MINUS, "-", line, column));
else if (char === "*") this.tokens.push(new Token(TokenType.STAR, "*", line, column));
else if (char === "/") this.tokens.push(new Token(TokenType.SLASH, "/", line, column));
else if (char === "%") this.tokens.push(new Token(TokenType.PERCENT, "%", line, column));
else if (char === "=") if (this.peek() === "=") {
this.advance();
this.tokens.push(new Token(TokenType.EQ, "==", line, column));
} else if (this.peek() === ">") {
this.advance();
this.tokens.push(new Token(TokenType.FAT_ARROW, "=>", line, column));
} else this.tokens.push(new Token(TokenType.ASSIGN, "=", line, column));
else if (char === "!") if (this.peek() === "=") {
this.advance();
this.tokens.push(new Token(TokenType.NEQ, "!=", line, column));
} else this.tokens.push(new Token(TokenType.NOT, "!", line, column));
else if (char === "<") if (this.peek() === "=") {
this.advance();
this.tokens.push(new Token(TokenType.LTE, "<=", line, column));
} else this.tokens.push(new Token(TokenType.LT, "<", line, column));
else if (char === ">") if (this.peek() === "=") {
this.advance();
this.tokens.push(new Token(TokenType.GTE, ">=", line, column));
} else this.tokens.push(new Token(TokenType.GT, ">", line, column));
else if (char === "&") if (this.peek() === "&") {
this.advance();
this.tokens.push(new Token(TokenType.AND, "&&", line, column));
} else this.tokens.push(new Token(TokenType.ERROR, `Unexpected char: &`, line, column));
else if (char === "|") if (this.peek() === "|") {
this.advance();
this.tokens.push(new Token(TokenType.OR, "||", line, column));
} else this.tokens.push(new Token(TokenType.ERROR, `Unexpected char: |`, line, column));
else if (char === "(") this.tokens.push(new Token(TokenType.LPAREN, "(", line, column));
else if (char === ")") this.tokens.push(new Token(TokenType.RPAREN, ")", line, column));
else if (char === "{") this.tokens.push(new Token(TokenType.LBRACE, "{", line, column));
else if (char === "}") this.tokens.push(new Token(TokenType.RBRACE, "}", line, column));
else if (char === "[") this.tokens.push(new Token(TokenType.LBRACKET, "[", line, column));
else if (char === "]") this.tokens.push(new Token(TokenType.RBRACKET, "]", line, column));
else if (char === ":") this.tokens.push(new Token(TokenType.COLON, ":", line, column));
else if (char === ";") this.tokens.push(new Token(TokenType.SEMICOLON, ";", line, column));
else if (char === ",") this.tokens.push(new Token(TokenType.COMMA, ",", line, column));
else if (char === ".") this.tokens.push(new Token(TokenType.DOT, ".", line, column));
else this.tokens.push(new Token(TokenType.ERROR, `Unexpected char: ${char}`, line, column));
}
this.tokens.push(new Token(TokenType.EOF, "", this.line, this.column));
return this.tokens;
}
};
//#endregion
exports.Lexer = Lexer;
exports.Token = Token;
exports.TokenType = TokenType;
//# sourceMappingURL=lexer.cjs.map