@gmsoft/spel2js/test/spec/Tokenizer.spec.js

Parse Spring Expression Language in JavaScript

import { Tokenizer } from '../../src/Tokenizer';
import { TokenKind } from '../../src/TokenKind';

describe('tokenizer', () => {
  let tokenize = Tokenizer.tokenize;

  beforeEach(() => {
    // add spies
  });

  it('should return an array of one Int token', () => {
    //when
    let tokens = tokenize('123');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_INT);
    expect(tokens[0].stringValue()).toBe('123');
  });

  it('should return an array of one long token', () => {
    //when
    let tokens = tokenize('123l');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_LONG);
    expect(tokens[0].stringValue()).toBe('123');
  });

  it('should return an array of one hex int token', () => {
    //when
    let tokens = tokenize('0xFF');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_HEXINT);
    expect(tokens[0].stringValue()).toBe('FF');
  });

  it('should return an array of one hex long token', () => {
    //when
    let tokens = tokenize('0xFFl');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_HEXLONG);
    expect(tokens[0].stringValue()).toBe('FF');
  });

  it('should return an array of one string token', () => {
    //when
    let tokens = tokenize("'hello world!'");

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_STRING);
    expect(tokens[0].stringValue()).toBe("'hello world!'");
  });

  it('should return an array of one real token', () => {
    //when
    let tokens = tokenize('123.4');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_REAL);
    expect(tokens[0].stringValue()).toBe('123.4');
  });

  it('should return an array of one real float token', () => {
    //when
    let tokens = tokenize('123.4f');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(1);
    expect(tokens[0].getKind()).toBe(TokenKind.LITERAL_REAL_FLOAT);
    expect(tokens[0].stringValue()).toBe('123.4f');
  });

  //more complex expressions
  it('should return an array of 4 tokens when given a function with 1 arg', () => {
    //when
    let tokens = tokenize("hasPermission('DISCUSSION_POST')");

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(4);
    expect(tokens[0].getKind()).toBe(TokenKind.IDENTIFIER);
    expect(tokens[1].getKind()).toBe(TokenKind.LPAREN);
    expect(tokens[2].getKind()).toBe(TokenKind.LITERAL_STRING);
    expect(tokens[3].getKind()).toBe(TokenKind.RPAREN);
  });

  it('should return an array of 11 tokens when given comparison of principal to arg property', () => {
    //when
    let tokens = tokenize("principal.email == #comment.user['email']");

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(11);
    expect(tokens[0].getKind()).toBe(TokenKind.IDENTIFIER);
    expect(tokens[1].getKind()).toBe(TokenKind.DOT);
    expect(tokens[2].getKind()).toBe(TokenKind.IDENTIFIER);
    expect(tokens[3].getKind()).toBe(TokenKind.EQ);
    expect(tokens[4].getKind()).toBe(TokenKind.HASH);
    expect(tokens[5].getKind()).toBe(TokenKind.IDENTIFIER);
    expect(tokens[6].getKind()).toBe(TokenKind.DOT);
    expect(tokens[7].getKind()).toBe(TokenKind.IDENTIFIER);
    expect(tokens[8].getKind()).toBe(TokenKind.LSQUARE);
    expect(tokens[9].getKind()).toBe(TokenKind.LITERAL_STRING);
    expect(tokens[10].getKind()).toBe(TokenKind.RSQUARE);
  });

  it('should tokenize unary operators', () => {
    //when
    let tokens = tokenize('+ - * / ^ % ++ -- ! =');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(10);
    expect(tokens[0].getKind()).toBe(TokenKind.PLUS);
    expect(tokens[1].getKind()).toBe(TokenKind.MINUS);
    expect(tokens[2].getKind()).toBe(TokenKind.STAR);
    expect(tokens[3].getKind()).toBe(TokenKind.DIV);
    expect(tokens[4].getKind()).toBe(TokenKind.POWER);
    expect(tokens[5].getKind()).toBe(TokenKind.MOD);
    expect(tokens[6].getKind()).toBe(TokenKind.INC);
    expect(tokens[7].getKind()).toBe(TokenKind.DEC);
    expect(tokens[8].getKind()).toBe(TokenKind.NOT);
    expect(tokens[9].getKind()).toBe(TokenKind.ASSIGN);
  });

  it('should tokenize comparison operators', () => {
    //when
    let tokens = tokenize('>= > <= < == != && ||');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(8);
    expect(tokens[0].getKind()).toBe(TokenKind.GE);
    expect(tokens[1].getKind()).toBe(TokenKind.GT);
    expect(tokens[2].getKind()).toBe(TokenKind.LE);
    expect(tokens[3].getKind()).toBe(TokenKind.LT);
    expect(tokens[4].getKind()).toBe(TokenKind.EQ);
    expect(tokens[5].getKind()).toBe(TokenKind.NE);
    expect(tokens[6].getKind()).toBe(TokenKind.SYMBOLIC_AND);
    expect(tokens[7].getKind()).toBe(TokenKind.SYMBOLIC_OR);
  });

  //this test fails. The tokenizer from Spring does not support these
  /*it('should tokenize keywords', ()=>{
        //when
        let tokens = tokenize('instanceOf matches between');

        //then
        expect(tokens).toBeDefined();
        expect(tokens.length).toBe(3);
        expect(tokens[0].getKind()).toBe(TokenKind.INSTANCEOF);
        expect(tokens[1].getKind()).toBe(TokenKind.MATCHES);
        expect(tokens[2].getKind()).toBe(TokenKind.BETWEEN);
    });*/

  it('should tokenize regex tokens', () => {
    //when
    let tokens = tokenize('^[ $[ ? ![ ?[');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(5);
    expect(tokens[0].getKind()).toBe(TokenKind.SELECT_FIRST);
    expect(tokens[1].getKind()).toBe(TokenKind.SELECT_LAST);
    expect(tokens[2].getKind()).toBe(TokenKind.QMARK);
    expect(tokens[3].getKind()).toBe(TokenKind.PROJECT);
    expect(tokens[4].getKind()).toBe(TokenKind.SELECT);
  });

  it('should tokenize miscellaneous tokens', () => {
    //when
    let tokens = tokenize(', : { } ?: ?. @');

    //then
    expect(tokens).toBeDefined();
    expect(tokens.length).toBe(7);
    expect(tokens[0].getKind()).toBe(TokenKind.COMMA);
    expect(tokens[1].getKind()).toBe(TokenKind.COLON);
    expect(tokens[2].getKind()).toBe(TokenKind.LCURLY);
    expect(tokens[3].getKind()).toBe(TokenKind.RCURLY);
    expect(tokens[4].getKind()).toBe(TokenKind.ELVIS);
    expect(tokens[5].getKind()).toBe(TokenKind.SAFE_NAVI);
    expect(tokens[6].getKind()).toBe(TokenKind.BEAN_REF);
  });

  it('should throw exception if using bitwise operators', () => {
    //given
    let shouldThrow1 = () => {
      tokenize('|');
    };
    let shouldThrow2 = () => {
      tokenize('&');
    };

    //then
    expect(shouldThrow1).toThrow();
    expect(shouldThrow2).toThrow();
  });

  it('should throw exception if escape character is used', () => {
    //given
    let shouldThrow = () => {
      tokenize('\\hello');
    };

    //then
    expect(shouldThrow).toThrow();
  });

  it('should throw exception if unsupported character is used', () => {
    //given
    let shouldThrow = () => {
      tokenize('¶');
    };

    //then
    expect(shouldThrow).toThrow();
  });

  it('should throw exception if string literal is unterminated', () => {
    //given
    let shouldThrowSingleQuote = () => {
      tokenize("'this is an unterminated stateme");
    };
    let shouldThrowDoubleQuote = () => {
      tokenize('"this is an unterminated stateme');
    };

    //then
    expect(shouldThrowSingleQuote).toThrow();
    expect(shouldThrowDoubleQuote).toThrow();
  });

  it('should throw exception if long identifier used on real', () => {
    //given
    let shouldThrow = () => {
      tokenize('3.4L');
    };

    //then
    expect(shouldThrow).toThrow();
  });
});