solidity-antlr4/dist/parser/tokenizer.cjs

Solidity Lang Lexer and Parser by official ANTLR4 grammar

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.tokenizer = exports.defaultTokenizerOption = void 0;
var _base = require("../ast/base.cjs");
var _antlr = require("../antlr4/index.cjs");
var _errorListener = require("./error-listener.cjs");
const defaultTokenizerOption = exports.defaultTokenizerOption = {
  tolerant: false
};
const tokenizer = (source, _options = {}) => {
  let tokens = [];
  const options = Object.assign({}, defaultTokenizerOption, _options);
  const listener = new _errorListener.SolidityErrorListener();
  try {
    const input = _antlr.CharStreams.fromString(source);
    const lexer = new _antlr.SolidityLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(listener);
    tokens = lexer.getAllTokens().map(token => {
      return {
        text: token.text,
        type: _antlr.SolidityLexer.symbolicNames[token.type],
        range: [token.start, token.stop],
        position: _base.Position.create(token.line, token.column)
      };
    });
  } catch (error) {
    if (error instanceof _errorListener.ParseError) {} else {
      listener.errors.push(new _errorListener.ParseError(error.message || "unknown error"));
    }
  }
  if (!options.tolerant) listener.throws();
  return tokens;
};
exports.tokenizer = tokenizer;