@toml-tools/lexer/lib/api.js

TOML Lexer Implemented in JavaScript

import { Lexer } from "chevrotain";
import { tokensArray, tokensDictionary } from "./tokens.js";

const tomlLexer = new Lexer(tokensArray, {
  // Reducing the amount of position tracking can provide a small performance boost (<10%)
  // Likely best to keep the full info for better error position reporting and
  // to expose "fuller" ITokens from the Lexer.
  positionTracking: "full",
  ensureOptimizations: true,

  // Both TOML lineTerminators contain a "\n"
  // The "lineTerminatorCharacters" parameter is only used to flag patterns that **may**
  // contain line terminators, so this is still correct.
  lineTerminatorCharacters: ["\n"],
  lineTerminatorsPattern: /\n|\r\n/g,
});

function tokenize(text) {
  return tomlLexer.tokenize(text);
}

export { tokenize, tokensDictionary };