eggtended-js/test/tokenRegex-test.js

A extended version of the Egg programming language from the Eloquent Javascript book.

const tokenRegex = require("../lib/parser/tokenRegex.js");
const should = require("should");

describe("Regex for LP token", () => {
  it("should recognize ( as valid a token", () => {
    const expected = { type: "LP", value: "(", start: 1, end: 1 };
    tokenRegex.LP.reset()
      .exec("(")
      .should.be.eql(expected);
  });

  it("should treat { as a synonym for (", () => {
    const expected = { type: "LP", value: "{", start: 1, end: 1 };
    tokenRegex.LP.reset()
      .exec("{")
      .should.be.eql(expected);
  });

  it("should treat [ as a synonym for (", () => {
    const expected = { type: "LP", value: "[", start: 1, end: 1 };
    tokenRegex.LP.reset()
      .exec("[")
      .should.be.eql(expected);
  });
});

describe("Regex for RP token", () => {
  it("should recognize ) as valid a token", () => {
    const expected = { type: "RP", value: ")", start: 1, end: 1 };
    tokenRegex.RP.reset()
      .exec(")")
      .should.be.eql(expected);
  });

  it("should treat } as a synonym for )", () => {
    const expected = { type: "RP", value: "}", start: 1, end: 1 };
    tokenRegex.RP.reset()
      .exec("}")
      .should.be.eql(expected);
  });

  it("should treat ] as a synonym for )", () => {
    const expected = { type: "RP", value: "]", start: 1, end: 1 };
    tokenRegex.RP.reset()
      .exec("]")
      .should.be.eql(expected);
  });
});

describe("Regex for COMMA token", () => {
  it("should recognize , as valid a token", () => {
    const expected = { type: "COMMA", value: ",", start: 1, end: 1 };
    tokenRegex.COMMA.reset()
      .exec(",")
      .should.be.eql(expected);
  });

  it("should recognize the : as an alias for ,", () => {
    const expected = { type: "COMMA", value: ":", start: 1, end: 1 };
    tokenRegex.COMMA.reset()
      .exec(":")
      .should.be.eql(expected);

    // Only when is not followed by a =
    should.not.exists(tokenRegex.COMMA.reset().exec(":="));
  });
});

describe("Regex for WHITES token", () => {
  it("should recognize any quantity of space", () => {
    const expected = { type: "WHITE", value: "  ", start: 1, end: 2 };
    tokenRegex.WHITES.reset()
      .exec("  ")
      .should.be.eql(expected);
  });

  it("should recognize as white even strings with newlines", () => {
    const expected = { type: "WHITE", value: "  \n  ", start: 1, end: 5 };
    tokenRegex.WHITES.reset()
      .exec("  \n  ")
      .should.be.eql(expected);
  });

  it("should recognize as white one-line comments with # or ;", () => {
    let expected = { type: "WHITE", value: "# commen t", start: 1, end: 10 };
    tokenRegex.WHITES.reset()
      .exec("# commen t")
      .should.be.eql(expected);

    expected = { type: "WHITE", value: "; commen t", start: 1, end: 10 };
    tokenRegex.WHITES.reset()
      .exec("; commen t")
      .should.be.eql(expected);
  });

  it("should recognize as white multiline commments with /* */", () => {
    const expected = {
      type: "WHITE",
      value: "/* this is \n a multiline\n comment */",
      start: 1,
      end: 36
    };
    tokenRegex.WHITES.reset()
      .exec("/* this is \n a multiline\n comment */")
      .should.be.eql(expected);
  });
});

describe("Regex for STRING token", () => {
  it('should recognize anything between " " as valid a token', () => {
    const expected = {
      type: "STRING",
      value: "hello world",
      start: 3,
      end: 13
    };
    tokenRegex.STRING.reset()
      .exec('"hello world"')
      .should.be.eql(expected);
  });

  it('shouldn\'t recognize strings without " ". Those are words', () => {
    should.not.exists(tokenRegex.STRING.reset().exec("hello world"));
  });
});

describe("Regex for WORD token", () => {
  it("shouldn't include some special characters on the identifier", () => {
    const expected = { type: "WORD", value: "fun", start: 1, end: 3 };
    tokenRegex.WORD.reset()
      .exec("fun()")
      .should.be.eql(expected);
    tokenRegex.WORD.reset()
      .exec("fun[]")
      .should.be.eql(expected);
    tokenRegex.WORD.reset()
      .exec("fun{}")
      .should.be.eql(expected);
    tokenRegex.WORD.reset()
      .exec("fun, ")
      .should.be.eql(expected);
    tokenRegex.WORD.reset()
      .exec("fun:, ")
      .should.be.eql(expected);
  });

  it("should detect [] as a whole WORD, instead of LP, RP", () => {
    const expected = { type: "WORD", value: "[]", start: 1, end: 2 };
    tokenRegex.WORD.reset()
      .exec("[]")
      .should.be.eql(expected);
  });

  it("should detect := as a whole WORD; instead of COMMA, WORD", () => {
    const expected = { type: "WORD", value: ":=", start: 1, end: 2 };
    tokenRegex.WORD.reset()
      .exec(":=")
      .should.be.eql(expected);
  });
});

describe("Regex for NUMBER token", () => {
  it("should match integers, with and without sign", () => {
    // Normal int
    let expected = { type: "NUMBER", value: "323", start: 1, end: 3 };
    tokenRegex.NUMBER.reset()
      .exec("323")
      .should.be.eql(expected);

    // With positive sign
    expected = { type: "NUMBER", value: "+2", start: 1, end: 2 };
    tokenRegex.NUMBER.reset()
      .exec("+2")
      .should.be.eql(expected);

    // With negative sign
    expected = { type: "NUMBER", value: "-123", start: 1, end: 4 };
    tokenRegex.NUMBER.reset()
      .exec("-123")
      .should.be.eql(expected);

    // Signs can't go alone!!
    should.not.exists(tokenRegex.NUMBER.reset().exec("-"));
    should.not.exists(tokenRegex.NUMBER.reset().exec("+"));
  });

  it("should match for floats, floating point and exponential notation", () => {
    // Normal float
    let expected = { type: "NUMBER", value: "12.34", start: 1, end: 5 };
    tokenRegex.NUMBER.reset()
      .exec("12.34")
      .should.be.eql(expected);

    // Float starting with .
    expected = { type: "NUMBER", value: ".35", start: 1, end: 3 };
    tokenRegex.NUMBER.reset()
      .exec(".35")
      .should.be.eql(expected);

    // Float in exponential notation
    expected = { type: "NUMBER", value: "2e5", start: 1, end: 3 };
    tokenRegex.NUMBER.reset()
      .exec("2e5")
      .should.be.eql(expected);

    // Float with decimals and exponential notation
    expected = { type: "NUMBER", value: "45.8e5", start: 1, end: 6 };
    tokenRegex.NUMBER.reset()
      .exec("45.8e5")
      .should.be.eql(expected);

    // Float with exponential notation with sign and E
    expected = { type: "NUMBER", value: "8E-34", start: 1, end: 5 };
    tokenRegex.NUMBER.reset()
      .exec("8E-34")
      .should.be.eql(expected);

    // Shouldn't match exponential alone or without exponential num
    should.not.exists(tokenRegex.NUMBER.reset().exec("e45"));
    should.not.exists(tokenRegex.NUMBER.reset().exec("E"));
  });
});