esther-medina-quintero-parser-nearley/src/tokens.js

Lab for language processors

const SPACE = /(?<SPACE>\s+|#.*|\/[*](?:.|\n)*?[*]\/)/;
const NUMBER = /(?<NUMBER>[-+]?\d+\.?\d*(?:[eE][-+]?\d+)?)/; NUMBER.value = x => Number(x);
const STRING = /(?<STRING>"(?:[^"\\]|\\.)*")/;
const WORD = /(?<WORD>[^\s(),"]+)/;
const LP = /(?<LP>\()/;
const RP = /(?<RP>\))/;
const COMMA = /(?<COMMA>,)/;
const ARRAY = /(?<ARRAY>\[.*\])/;

/** Tokens object: definitions */
const tokens = [
  SPACE,
  NUMBER,
  STRING,
  WORD,
  LP,
  RP,
  COMMA,
  ARRAY
];

module.exports = tokens;