solidity-antlr4/dist/parser/tokenizer.js

Solidity Lang Lexer and Parser by official ANTLR4 grammar

import { Position } from "../ast/base.js";
import { SolidityLexer, CharStreams } from "../antlr4/index.js";
import { ParseError, SolidityErrorListener } from "./error-listener.js";
export const defaultTokenizerOption = {
  tolerant: false
};
export const tokenizer = (source, _options = {}) => {
  let tokens = [];
  const options = Object.assign({}, defaultTokenizerOption, _options);
  const listener = new SolidityErrorListener();
  try {
    const input = CharStreams.fromString(source);
    const lexer = new SolidityLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(listener);
    tokens = lexer.getAllTokens().map((token) => {
      return {
        text: token.text,
        type: SolidityLexer.symbolicNames[token.type],
        range: [token.start, token.stop],
        position: Position.create(token.line, token.column)
      };
    });
  } catch (error) {
    if (error instanceof ParseError) {
    } else {
      listener.errors.push(new ParseError(error.message || "unknown error"));
    }
  }
  if (!options.tolerant)
    listener.throws();
  return tokens;
};