@ordojs/core/src/compiler/lexer-focused.test.ts

Core compiler and runtime for OrdoJS framework

/**
 * @fileoverview Focused OrdoJS Lexer Tests - Key functionality tests
 */

import { describe, expect, it } from 'vitest';
import { LexicalError, TokenType } from '../types/index.js';
import { OrdoJSLexer } from './lexer.js';

describe('OrdoJSLexer - Focused Tests', () => {
  describe('Basic Tokenization', () => {
    it('should tokenize keywords correctly', () => {
      const lexer = new OrdoJSLexer('component client server markup let const');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.COMPONENT);
      expect(stream.tokens[1].type).toBe(TokenType.CLIENT);
      expect(stream.tokens[2].type).toBe(TokenType.SERVER);
      expect(stream.tokens[3].type).toBe(TokenType.MARKUP);
      expect(stream.tokens[4].type).toBe(TokenType.LET);
      expect(stream.tokens[5].type).toBe(TokenType.CONST);
    });

    it('should tokenize operators correctly', () => {
      const lexer = new OrdoJSLexer('+ - * / = == != < >');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.PLUS);
      expect(stream.tokens[1].type).toBe(TokenType.MINUS);
      expect(stream.tokens[2].type).toBe(TokenType.MULTIPLY);
      expect(stream.tokens[3].type).toBe(TokenType.DIVIDE);
      expect(stream.tokens[4].type).toBe(TokenType.ASSIGN);
      expect(stream.tokens[5].type).toBe(TokenType.EQUALS);
      expect(stream.tokens[6].type).toBe(TokenType.NOT_EQUALS);
      expect(stream.tokens[7].type).toBe(TokenType.LESS_THAN);
      expect(stream.tokens[8].type).toBe(TokenType.GREATER_THAN);
    });

    it('should tokenize delimiters correctly', () => {
      const lexer = new OrdoJSLexer('{ } ( ) [ ]');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.LEFT_BRACE);
      expect(stream.tokens[1].type).toBe(TokenType.RIGHT_BRACE);
      expect(stream.tokens[2].type).toBe(TokenType.LEFT_PAREN);
      expect(stream.tokens[3].type).toBe(TokenType.RIGHT_PAREN);
      expect(stream.tokens[4].type).toBe(TokenType.LEFT_BRACKET);
      expect(stream.tokens[5].type).toBe(TokenType.RIGHT_BRACKET);
    });
  });

  describe('Literals', () => {
    it('should tokenize string literals', () => {
      const lexer = new OrdoJSLexer('"hello world"');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.STRING);
      expect(stream.tokens[0].value).toBe('hello world');
    });

    it('should tokenize numeric literals', () => {
      const lexer = new OrdoJSLexer('123 3.14');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.NUMBER);
      expect(stream.tokens[0].value).toBe('123');
      expect(stream.tokens[1].type).toBe(TokenType.NUMBER);
      expect(stream.tokens[1].value).toBe('3.14');
    });

    it('should tokenize boolean literals', () => {
      const lexer = new OrdoJSLexer('true false');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.BOOLEAN);
      expect(stream.tokens[0].value).toBe('true');
      expect(stream.tokens[1].type).toBe(TokenType.BOOLEAN);
      expect(stream.tokens[1].value).toBe('false');
    });
  });

  describe('Comments', () => {
    it('should tokenize line comments', () => {
      const lexer = new OrdoJSLexer('// this is a comment');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.COMMENT);
      expect(stream.tokens[0].value).toBe('// this is a comment');
    });

    it('should tokenize block comments', () => {
      const lexer = new OrdoJSLexer('/* block comment */');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.COMMENT);
      expect(stream.tokens[0].value).toBe('/* block comment */');
    });
  });

  describe('Simple Component Structure', () => {
    it('should tokenize basic component declaration', () => {
      const source = 'component MyComponent { }';
      const lexer = new OrdoJSLexer(source);
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.COMPONENT);
      expect(stream.tokens[1].type).toBe(TokenType.IDENTIFIER);
      expect(stream.tokens[1].value).toBe('MyComponent');
      expect(stream.tokens[2].type).toBe(TokenType.LEFT_BRACE);
      expect(stream.tokens[3].type).toBe(TokenType.RIGHT_BRACE);
    });

    it('should tokenize client block', () => {
      const source = 'client { let x = 5; }';
      const lexer = new OrdoJSLexer(source);
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.CLIENT);
      expect(stream.tokens[1].type).toBe(TokenType.LEFT_BRACE);
      expect(stream.tokens[2].type).toBe(TokenType.LET);
      expect(stream.tokens[3].type).toBe(TokenType.IDENTIFIER);
      expect(stream.tokens[3].value).toBe('x');
      expect(stream.tokens[4].type).toBe(TokenType.ASSIGN);
      expect(stream.tokens[5].type).toBe(TokenType.NUMBER);
      expect(stream.tokens[5].value).toBe('5');
    });

    it('should tokenize server block with public function', () => {
      const source = 'server { public function test() { } }';
      const lexer = new OrdoJSLexer(source);
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.SERVER);
      expect(stream.tokens[2].type).toBe(TokenType.PUBLIC);
      expect(stream.tokens[3].type).toBe(TokenType.IDENTIFIER);
      expect(stream.tokens[3].value).toBe('function');
      expect(stream.tokens[4].type).toBe(TokenType.IDENTIFIER);
      expect(stream.tokens[4].value).toBe('test');
    });
  });

  describe('Source Position Tracking', () => {
    it('should track line and column positions', () => {
      const source = 'let x = 5;\nlet y = 10;';
      const lexer = new OrdoJSLexer(source);
      const stream = lexer.tokenize();

      // First line
      expect(stream.tokens[0].position.line).toBe(1); // let
      expect(stream.tokens[0].position.column).toBe(1);

      // Find newline token
      const newlineIndex = stream.tokens.findIndex(t => t.type === TokenType.NEWLINE);
      expect(newlineIndex).toBeGreaterThan(-1);

      // Token after newline should be on line 2
      const tokenAfterNewline = stream.tokens[newlineIndex + 1];
      if (tokenAfterNewline && tokenAfterNewline.type === TokenType.LET) {
        expect(tokenAfterNewline.position.line).toBe(2);
      }
    });

    it('should track source ranges', () => {
      const lexer = new OrdoJSLexer('hello');
      const stream = lexer.tokenize();

      const token = stream.tokens[0];
      expect(token.range.start.offset).toBe(0);
      expect(token.range.end.offset).toBe(5);
    });
  });

  describe('Error Handling', () => {
    it('should throw error for unterminated string', () => {
      const lexer = new OrdoJSLexer('"unterminated');

      expect(() => lexer.tokenize()).toThrow(LexicalError);
    });

    it('should throw error for invalid characters', () => {
      const lexer = new OrdoJSLexer('let x = @;');

      expect(() => lexer.tokenize()).toThrow(LexicalError);
    });
  });

  describe('TokenStream Interface', () => {
    it('should implement peek() correctly', () => {
      const lexer = new OrdoJSLexer('let x');
      const stream = lexer.tokenize();

      expect(stream.peek().type).toBe(TokenType.LET);
      expect(stream.peek().type).toBe(TokenType.LET); // Should not advance
    });

    it('should implement advance() correctly', () => {
      const lexer = new OrdoJSLexer('let x');
      const stream = lexer.tokenize();

      const first = stream.advance();
      expect(first.type).toBe(TokenType.LET);

      const second = stream.peek();
      expect(second.type).toBe(TokenType.IDENTIFIER);
      expect(second.value).toBe('x');
    });

    it('should implement isAtEnd() correctly', () => {
      const lexer = new OrdoJSLexer('x');
      const stream = lexer.tokenize();

      expect(stream.isAtEnd()).toBe(false);

      stream.advance(); // x
      expect(stream.isAtEnd()).toBe(true); // EOF
    });
  });

  describe('Increment/Decrement Operators', () => {
    it('should tokenize increment and decrement operators', () => {
      const lexer = new OrdoJSLexer('count++ --value');
      const stream = lexer.tokenize();

      expect(stream.tokens[0].type).toBe(TokenType.IDENTIFIER);
      expect(stream.tokens[0].value).toBe('count');
      expect(stream.tokens[1].type).toBe(TokenType.INCREMENT);
      expect(stream.tokens[2].type).toBe(TokenType.DECREMENT);
      expect(stream.tokens[3].type).toBe(TokenType.IDENTIFIER);
      expect(stream.tokens[3].value).toBe('value');
    });
  });
});