UNPKG

zignet

Version:

MCP server for Zig — AI-powered code analysis, validation, and documentation with fine-tuned LLM

1 lines 17.4 kB
{"version":3,"file":"lexer.cjs","names":["type: TokenType","value: string","line: number","column: number","KEYWORDS: Record<string, TokenType>","ESCAPE_SEQUENCES: Record<string, string>","source: string"],"sources":["../src/lexer.ts"],"sourcesContent":["/**\n * ZigNet Lexer\n * Tokenizes Zig source code\n */\n\nexport enum TokenType {\n // Keywords\n FN = 'FN',\n CONST = 'CONST',\n VAR = 'VAR',\n STRUCT = 'STRUCT',\n UNION = 'UNION',\n ENUM = 'ENUM',\n IF = 'IF',\n ELSE = 'ELSE',\n WHILE = 'WHILE',\n FOR = 'FOR',\n RETURN = 'RETURN',\n BREAK = 'BREAK',\n CONTINUE = 'CONTINUE',\n COMPTIME = 'COMPTIME',\n INLINE = 'INLINE',\n\n // Types\n I32 = 'I32',\n I64 = 'I64',\n U32 = 'U32',\n F32 = 'F32',\n F64 = 'F64',\n BOOL = 'BOOL',\n VOID = 'VOID',\n\n // Literals\n NUMBER = 'NUMBER',\n STRING = 'STRING',\n IDENT = 'IDENT',\n TRUE = 'TRUE',\n FALSE = 'FALSE',\n\n // Operators\n PLUS = 'PLUS',\n MINUS = 'MINUS',\n STAR = 'STAR',\n SLASH = 'SLASH',\n PERCENT = 'PERCENT',\n EQ = 'EQ',\n NEQ = 'NEQ',\n LT = 'LT',\n GT = 'GT',\n LTE = 'LTE',\n GTE = 'GTE',\n ASSIGN = 'ASSIGN',\n PLUS_ASSIGN = 'PLUS_ASSIGN',\n AND = 'AND',\n OR = 'OR',\n NOT = 'NOT',\n\n // Punctuation\n LPAREN = 'LPAREN',\n RPAREN = 'RPAREN',\n LBRACE = 'LBRACE',\n RBRACE = 'RBRACE',\n LBRACKET = 'LBRACKET',\n RBRACKET = 'RBRACKET',\n COLON = 'COLON',\n SEMICOLON = 'SEMICOLON',\n COMMA = 'COMMA',\n DOT = 'DOT',\n ARROW = 'ARROW',\n FAT_ARROW = 'FAT_ARROW',\n\n // Special\n EOF = 'EOF',\n ERROR = 'ERROR',\n}\n\nexport class Token {\n constructor(\n public type: TokenType,\n public value: string,\n public line: number,\n public column: number\n ) {}\n\n toString(): string {\n return `Token(${this.type}, \"${this.value}\", ${this.line}:${this.column})`;\n }\n}\n\nconst KEYWORDS: Record<string, TokenType> = {\n fn: TokenType.FN,\n const: TokenType.CONST,\n var: TokenType.VAR,\n struct: TokenType.STRUCT,\n union: TokenType.UNION,\n enum: TokenType.ENUM,\n if: TokenType.IF,\n else: TokenType.ELSE,\n while: TokenType.WHILE,\n for: TokenType.FOR,\n return: TokenType.RETURN,\n break: TokenType.BREAK,\n continue: TokenType.CONTINUE,\n comptime: TokenType.COMPTIME,\n inline: TokenType.INLINE,\n i32: TokenType.I32,\n i64: TokenType.I64,\n u32: TokenType.U32,\n f32: TokenType.F32,\n f64: TokenType.F64,\n bool: TokenType.BOOL,\n void: TokenType.VOID,\n true: TokenType.TRUE,\n false: TokenType.FALSE,\n};\n\nconst ESCAPE_SEQUENCES: Record<string, string> = {\n n: '\\n',\n t: '\\t',\n r: '\\r',\n '\\\\': '\\\\',\n '\"': '\"',\n \"'\": \"'\",\n};\n\nexport class Lexer {\n private position = 0;\n private line = 1;\n private column = 1;\n private tokens: Token[] = [];\n\n constructor(private source: string) {}\n\n private error(message: string): Token {\n return new Token(TokenType.ERROR, message, this.line, this.column);\n }\n\n private peek(offset = 0): string {\n const pos = this.position + offset;\n if (pos >= this.source.length) return '\\0';\n return this.source[pos];\n }\n\n private advance(): string {\n const char = this.source[this.position];\n this.position++;\n if (char === '\\n') {\n this.line++;\n this.column = 1;\n } else {\n this.column++;\n }\n return char;\n }\n\n private skipWhitespace(): void {\n while (this.position < this.source.length && /\\s/.test(this.peek())) {\n this.advance();\n }\n }\n\n private skipComment(): boolean {\n if (this.peek() === '/' && this.peek(1) === '/') {\n while (this.peek() !== '\\n' && this.peek() !== '\\0') {\n this.advance();\n }\n return true;\n }\n return false;\n }\n\n private readIdentifier(): string {\n let value = '';\n while (/[a-zA-Z0-9_]/.test(this.peek())) {\n value += this.advance();\n }\n return value;\n }\n\n private readNumber(): string {\n let value = '';\n while (/[0-9]/.test(this.peek())) {\n value += this.advance();\n }\n if (this.peek() === '.' && /[0-9]/.test(this.peek(1))) {\n value += this.advance(); // '.'\n while (/[0-9]/.test(this.peek())) {\n value += this.advance();\n }\n }\n return value;\n }\n\n private readString(quote: string): string | Token {\n let value = '';\n this.advance(); // opening quote\n while (this.peek() !== quote && this.peek() !== '\\0') {\n if (this.peek() === '\\\\') {\n this.advance();\n const escaped = this.advance();\n value += this.getEscapeSequence(escaped);\n } else {\n value += this.advance();\n }\n }\n if (this.peek() === quote) {\n this.advance(); // closing quote\n } else {\n return this.error('Unterminated string');\n }\n return value;\n }\n\n private getEscapeSequence(char: string): string {\n return ESCAPE_SEQUENCES[char] || char;\n }\n\n tokenize(): Token[] {\n while (this.position < this.source.length) {\n this.skipWhitespace();\n if (this.skipComment()) continue;\n\n const char = this.peek();\n const line = this.line;\n const column = this.column;\n\n if (char === '\\0') break;\n\n // Numbers\n if (/[0-9]/.test(char)) {\n const value = this.readNumber();\n this.tokens.push(new Token(TokenType.NUMBER, value, line, column));\n continue;\n }\n\n // Strings\n if (char === '\"' || char === \"'\") {\n const value = this.readString(char);\n if (value instanceof Token) {\n this.tokens.push(value);\n } else {\n this.tokens.push(new Token(TokenType.STRING, value, line, column));\n }\n continue;\n }\n\n // Identifiers and keywords\n if (/[a-zA-Z_]/.test(char)) {\n const value = this.readIdentifier();\n const type = KEYWORDS[value] || TokenType.IDENT;\n this.tokens.push(new Token(type, value, line, column));\n continue;\n }\n\n // Operators and punctuation\n this.advance();\n if (char === '+') {\n if (this.peek() === '=') {\n this.advance();\n this.tokens.push(new Token(TokenType.PLUS_ASSIGN, '+=', line, column));\n } else {\n this.tokens.push(new Token(TokenType.PLUS, '+', line, column));\n }\n } else if (char === '-') {\n if (this.peek() === '>') {\n this.advance();\n this.tokens.push(new Token(TokenType.ARROW, '->', line, column));\n } else {\n this.tokens.push(new Token(TokenType.MINUS, '-', line, column));\n }\n } else if (char === '*') {\n this.tokens.push(new Token(TokenType.STAR, '*', line, column));\n } else if (char === '/') {\n this.tokens.push(new Token(TokenType.SLASH, '/', line, column));\n } else if (char === '%') {\n this.tokens.push(new Token(TokenType.PERCENT, '%', line, column));\n } else if (char === '=') {\n if (this.peek() === '=') {\n this.advance();\n this.tokens.push(new Token(TokenType.EQ, '==', line, column));\n } else if (this.peek() === '>') {\n this.advance();\n this.tokens.push(new Token(TokenType.FAT_ARROW, '=>', line, column));\n } else {\n this.tokens.push(new Token(TokenType.ASSIGN, '=', line, column));\n }\n } else if (char === '!') {\n if (this.peek() === '=') {\n this.advance();\n this.tokens.push(new Token(TokenType.NEQ, '!=', line, column));\n } else {\n this.tokens.push(new Token(TokenType.NOT, '!', line, column));\n }\n } else if (char === '<') {\n if (this.peek() === '=') {\n this.advance();\n this.tokens.push(new Token(TokenType.LTE, '<=', line, column));\n } else {\n this.tokens.push(new Token(TokenType.LT, '<', line, column));\n }\n } else if (char === '>') {\n if (this.peek() === '=') {\n this.advance();\n this.tokens.push(new Token(TokenType.GTE, '>=', line, column));\n } else {\n this.tokens.push(new Token(TokenType.GT, '>', line, column));\n }\n } else if (char === '&') {\n if (this.peek() === '&') {\n this.advance();\n this.tokens.push(new Token(TokenType.AND, '&&', line, column));\n } else {\n this.tokens.push(\n new Token(TokenType.ERROR, `Unexpected char: &`, line, column)\n );\n }\n } else if (char === '|') {\n if (this.peek() === '|') {\n this.advance();\n this.tokens.push(new Token(TokenType.OR, '||', line, column));\n } else {\n this.tokens.push(\n new Token(TokenType.ERROR, `Unexpected char: |`, line, column)\n );\n }\n } else if (char === '(') {\n this.tokens.push(new Token(TokenType.LPAREN, '(', line, column));\n } else if (char === ')') {\n this.tokens.push(new Token(TokenType.RPAREN, ')', line, column));\n } else if (char === '{') {\n this.tokens.push(new Token(TokenType.LBRACE, '{', line, column));\n } else if (char === '}') {\n this.tokens.push(new Token(TokenType.RBRACE, '}', line, column));\n } else if (char === '[') {\n this.tokens.push(new Token(TokenType.LBRACKET, '[', line, column));\n } else if (char === ']') {\n this.tokens.push(new Token(TokenType.RBRACKET, ']', line, column));\n } else if (char === ':') {\n this.tokens.push(new Token(TokenType.COLON, ':', line, column));\n } else if (char === ';') {\n this.tokens.push(new Token(TokenType.SEMICOLON, ';', line, column));\n } else if (char === ',') {\n this.tokens.push(new Token(TokenType.COMMA, ',', line, column));\n } else if (char === '.') {\n this.tokens.push(new Token(TokenType.DOT, '.', line, column));\n } else {\n this.tokens.push(\n new Token(TokenType.ERROR, `Unexpected char: ${char}`, line, column)\n );\n }\n }\n\n this.tokens.push(new Token(TokenType.EOF, '', this.line, this.column));\n return this.tokens;\n }\n}\n"],"mappings":";;;;;;AAKA,IAAY,kDAAL;AAEH;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;AACA;AACA;AACA;AACA;AAGA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;AACA;;;AAGJ,IAAa,QAAb,MAAmB;CACf,YACI,AAAOA,MACP,AAAOC,OACP,AAAOC,MACP,AAAOC,QACT;EAJS;EACA;EACA;EACA;;CAGX,WAAmB;AACf,SAAO,SAAS,KAAK,KAAK,KAAK,KAAK,MAAM,KAAK,KAAK,KAAK,GAAG,KAAK,OAAO;;;AAIhF,MAAMC,WAAsC;CACxC,IAAI,UAAU;CACd,OAAO,UAAU;CACjB,KAAK,UAAU;CACf,QAAQ,UAAU;CAClB,OAAO,UAAU;CACjB,MAAM,UAAU;CAChB,IAAI,UAAU;CACd,MAAM,UAAU;CAChB,OAAO,UAAU;CACjB,KAAK,UAAU;CACf,QAAQ,UAAU;CAClB,OAAO,UAAU;CACjB,UAAU,UAAU;CACpB,UAAU,UAAU;CACpB,QAAQ,UAAU;CAClB,KAAK,UAAU;CACf,KAAK,UAAU;CACf,KAAK,UAAU;CACf,KAAK,UAAU;CACf,KAAK,UAAU;CACf,MAAM,UAAU;CAChB,MAAM,UAAU;CAChB,MAAM,UAAU;CAChB,OAAO,UAAU;CACpB;AAED,MAAMC,mBAA2C;CAC7C,GAAG;CACH,GAAG;CACH,GAAG;CACH,MAAM;CACN,MAAK;CACL,KAAK;CACR;AAED,IAAa,QAAb,MAAmB;CACf,AAAQ,WAAW;CACnB,AAAQ,OAAO;CACf,AAAQ,SAAS;CACjB,AAAQ,SAAkB,EAAE;CAE5B,YAAY,AAAQC,QAAgB;EAAhB;;CAEpB,AAAQ,MAAM,SAAwB;AAClC,SAAO,IAAI,MAAM,UAAU,OAAO,SAAS,KAAK,MAAM,KAAK,OAAO;;CAGtE,AAAQ,KAAK,SAAS,GAAW;EAC7B,MAAM,MAAM,KAAK,WAAW;AAC5B,MAAI,OAAO,KAAK,OAAO,OAAQ,QAAO;AACtC,SAAO,KAAK,OAAO;;CAGvB,AAAQ,UAAkB;EACtB,MAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,OAAK;AACL,MAAI,SAAS,MAAM;AACf,QAAK;AACL,QAAK,SAAS;QAEd,MAAK;AAET,SAAO;;CAGX,AAAQ,iBAAuB;AAC3B,SAAO,KAAK,WAAW,KAAK,OAAO,UAAU,KAAK,KAAK,KAAK,MAAM,CAAC,CAC/D,MAAK,SAAS;;CAItB,AAAQ,cAAuB;AAC3B,MAAI,KAAK,MAAM,KAAK,OAAO,KAAK,KAAK,EAAE,KAAK,KAAK;AAC7C,UAAO,KAAK,MAAM,KAAK,QAAQ,KAAK,MAAM,KAAK,KAC3C,MAAK,SAAS;AAElB,UAAO;;AAEX,SAAO;;CAGX,AAAQ,iBAAyB;EAC7B,IAAI,QAAQ;AACZ,SAAO,eAAe,KAAK,KAAK,MAAM,CAAC,CACnC,UAAS,KAAK,SAAS;AAE3B,SAAO;;CAGX,AAAQ,aAAqB;EACzB,IAAI,QAAQ;AACZ,SAAO,QAAQ,KAAK,KAAK,MAAM,CAAC,CAC5B,UAAS,KAAK,SAAS;AAE3B,MAAI,KAAK,MAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,KAAK,EAAE,CAAC,EAAE;AACnD,YAAS,KAAK,SAAS;AACvB,UAAO,QAAQ,KAAK,KAAK,MAAM,CAAC,CAC5B,UAAS,KAAK,SAAS;;AAG/B,SAAO;;CAGX,AAAQ,WAAW,OAA+B;EAC9C,IAAI,QAAQ;AACZ,OAAK,SAAS;AACd,SAAO,KAAK,MAAM,KAAK,SAAS,KAAK,MAAM,KAAK,KAC5C,KAAI,KAAK,MAAM,KAAK,MAAM;AACtB,QAAK,SAAS;GACd,MAAM,UAAU,KAAK,SAAS;AAC9B,YAAS,KAAK,kBAAkB,QAAQ;QAExC,UAAS,KAAK,SAAS;AAG/B,MAAI,KAAK,MAAM,KAAK,MAChB,MAAK,SAAS;MAEd,QAAO,KAAK,MAAM,sBAAsB;AAE5C,SAAO;;CAGX,AAAQ,kBAAkB,MAAsB;AAC5C,SAAO,iBAAiB,SAAS;;CAGrC,WAAoB;AAChB,SAAO,KAAK,WAAW,KAAK,OAAO,QAAQ;AACvC,QAAK,gBAAgB;AACrB,OAAI,KAAK,aAAa,CAAE;GAExB,MAAM,OAAO,KAAK,MAAM;GACxB,MAAM,OAAO,KAAK;GAClB,MAAM,SAAS,KAAK;AAEpB,OAAI,SAAS,KAAM;AAGnB,OAAI,QAAQ,KAAK,KAAK,EAAE;IACpB,MAAM,QAAQ,KAAK,YAAY;AAC/B,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,OAAO,MAAM,OAAO,CAAC;AAClE;;AAIJ,OAAI,SAAS,QAAO,SAAS,KAAK;IAC9B,MAAM,QAAQ,KAAK,WAAW,KAAK;AACnC,QAAI,iBAAiB,MACjB,MAAK,OAAO,KAAK,MAAM;QAEvB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,OAAO,MAAM,OAAO,CAAC;AAEtE;;AAIJ,OAAI,YAAY,KAAK,KAAK,EAAE;IACxB,MAAM,QAAQ,KAAK,gBAAgB;IACnC,MAAM,OAAO,SAAS,UAAU,UAAU;AAC1C,SAAK,OAAO,KAAK,IAAI,MAAM,MAAM,OAAO,MAAM,OAAO,CAAC;AACtD;;AAIJ,QAAK,SAAS;AACd,OAAI,SAAS,IACT,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,aAAa,MAAM,MAAM,OAAO,CAAC;SAEtE,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,MAAM,KAAK,MAAM,OAAO,CAAC;YAE3D,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,OAAO,MAAM,MAAM,OAAO,CAAC;SAEhE,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,OAAO,CAAC;YAE5D,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,MAAM,KAAK,MAAM,OAAO,CAAC;YACvD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,OAAO,CAAC;YACxD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,SAAS,KAAK,MAAM,OAAO,CAAC;YAC1D,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,IAAI,MAAM,MAAM,OAAO,CAAC;cACtD,KAAK,MAAM,KAAK,KAAK;AAC5B,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,WAAW,MAAM,MAAM,OAAO,CAAC;SAEpE,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,KAAK,MAAM,OAAO,CAAC;YAE7D,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,MAAM,MAAM,OAAO,CAAC;SAE9D,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,KAAK,MAAM,OAAO,CAAC;YAE1D,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,MAAM,MAAM,OAAO,CAAC;SAE9D,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,IAAI,KAAK,MAAM,OAAO,CAAC;YAEzD,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,MAAM,MAAM,OAAO,CAAC;SAE9D,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,IAAI,KAAK,MAAM,OAAO,CAAC;YAEzD,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,MAAM,MAAM,OAAO,CAAC;SAE9D,MAAK,OAAO,KACR,IAAI,MAAM,UAAU,OAAO,sBAAsB,MAAM,OAAO,CACjE;YAEE,SAAS,IAChB,KAAI,KAAK,MAAM,KAAK,KAAK;AACrB,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,IAAI,MAAM,UAAU,IAAI,MAAM,MAAM,OAAO,CAAC;SAE7D,MAAK,OAAO,KACR,IAAI,MAAM,UAAU,OAAO,sBAAsB,MAAM,OAAO,CACjE;YAEE,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,KAAK,MAAM,OAAO,CAAC;YACzD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,KAAK,MAAM,OAAO,CAAC;YACzD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,KAAK,MAAM,OAAO,CAAC;YACzD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,QAAQ,KAAK,MAAM,OAAO,CAAC;YACzD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,UAAU,KAAK,MAAM,OAAO,CAAC;YAC3D,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,UAAU,KAAK,MAAM,OAAO,CAAC;YAC3D,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,OAAO,CAAC;YACxD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,WAAW,KAAK,MAAM,OAAO,CAAC;YAC5D,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,OAAO,CAAC;YACxD,SAAS,IAChB,MAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,KAAK,MAAM,OAAO,CAAC;OAE7D,MAAK,OAAO,KACR,IAAI,MAAM,UAAU,OAAO,oBAAoB,QAAQ,MAAM,OAAO,CACvE;;AAIT,OAAK,OAAO,KAAK,IAAI,MAAM,UAAU,KAAK,IAAI,KAAK,MAAM,KAAK,OAAO,CAAC;AACtE,SAAO,KAAK"}