@aurahelper/languages
Version:
Language Libraries to work with XML, Aura, Apex... files. tokenizers, parsers, system classes and much more
301 lines • 16.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Tokenizer = void 0;
var core_1 = require("@aurahelper/core");
var symbolTokens = {
">>>=": core_1.TokenTypes.OPERATOR.BITWISE.UNSIGNED_RIGHT_ASSIGN,
'<!--': core_1.TokenTypes.COMMENT.XML_START,
">>=": core_1.TokenTypes.OPERATOR.BITWISE.SIGNED_RIGTH_ASSIGN,
"<<=": core_1.TokenTypes.OPERATOR.BITWISE.LEFT_ASSIGN,
">>>": core_1.TokenTypes.OPERATOR.BITWISE.UNSIGNED_RIGHT,
"!==": core_1.TokenTypes.OPERATOR.LOGICAL.INEQUALITY_EXACT,
"===": core_1.TokenTypes.OPERATOR.LOGICAL.EQUALITY_EXACT,
'-->': core_1.TokenTypes.COMMENT.XML_END,
">>": core_1.TokenTypes.OPERATOR.BITWISE.SIGNED_RIGHT,
"<<": core_1.TokenTypes.OPERATOR.BITWISE.SIGNED_LEFT,
"^=": core_1.TokenTypes.OPERATOR.BITWISE.EXCLUSIVE_OR_ASSIGN,
"--": core_1.TokenTypes.OPERATOR.ARITHMETIC.DECREMENT,
"++": core_1.TokenTypes.OPERATOR.ARITHMETIC.INCREMENT,
"!=": core_1.TokenTypes.OPERATOR.LOGICAL.INEQUALITY,
"<>": core_1.TokenTypes.OPERATOR.LOGICAL.INEQUALITY,
"==": core_1.TokenTypes.OPERATOR.LOGICAL.EQUALITY,
"||": core_1.TokenTypes.OPERATOR.LOGICAL.OR,
"|=": core_1.TokenTypes.OPERATOR.LOGICAL.OR_ASSIGN,
"&&": core_1.TokenTypes.OPERATOR.LOGICAL.AND,
"&=": core_1.TokenTypes.OPERATOR.LOGICAL.AND_ASSIGN,
">=": core_1.TokenTypes.OPERATOR.LOGICAL.GREATER_THAN_EQUALS,
"<=": core_1.TokenTypes.OPERATOR.LOGICAL.LESS_THAN_EQUALS,
"=>": core_1.TokenTypes.OPERATOR.ASSIGN.MAP_KEY_VALUE,
"+=": core_1.TokenTypes.OPERATOR.ARITHMETIC.ADD_ASSIGN,
"-=": core_1.TokenTypes.OPERATOR.ARITHMETIC.SUBSTRACT_ASSIGN,
"*=": core_1.TokenTypes.OPERATOR.ARITHMETIC.MULTIPLY_ASSIGN,
"/=": core_1.TokenTypes.OPERATOR.ARITHMETIC.DIVIDE_ASSIGN,
"^": core_1.TokenTypes.OPERATOR.BITWISE.EXCLUSIVE_OR,
"|": core_1.TokenTypes.OPERATOR.BITWISE.OR,
"&": core_1.TokenTypes.OPERATOR.BITWISE.AND,
"+": core_1.TokenTypes.OPERATOR.ARITHMETIC.ADD,
"-": core_1.TokenTypes.OPERATOR.ARITHMETIC.SUBSTRACT,
"*": core_1.TokenTypes.OPERATOR.ARITHMETIC.MULTIPLY,
"/": core_1.TokenTypes.OPERATOR.ARITHMETIC.DIVIDE,
"!": core_1.TokenTypes.OPERATOR.LOGICAL.NOT,
"<": core_1.TokenTypes.OPERATOR.LOGICAL.LESS_THAN,
">": core_1.TokenTypes.OPERATOR.LOGICAL.GREATER_THAN,
"=": core_1.TokenTypes.OPERATOR.ASSIGN.ASSIGN,
"/**": core_1.TokenTypes.COMMENT.BLOCK_START,
"/*": core_1.TokenTypes.COMMENT.BLOCK_START,
"*/": core_1.TokenTypes.COMMENT.BLOCK_END,
"//": core_1.TokenTypes.COMMENT.LINE,
"///": core_1.TokenTypes.COMMENT.LINE_DOC,
"(": core_1.TokenTypes.OPERATOR.PRIORITY.PARENTHESIS_OPEN,
")": core_1.TokenTypes.OPERATOR.PRIORITY.PARENTHESIS_CLOSE,
"{": core_1.TokenTypes.BRACKET.CURLY_OPEN,
"}": core_1.TokenTypes.BRACKET.CURLY_CLOSE,
"[": core_1.TokenTypes.BRACKET.SQUARE_OPEN,
"]": core_1.TokenTypes.BRACKET.SQUARE_CLOSE,
",": core_1.TokenTypes.PUNCTUATION.COMMA,
";": core_1.TokenTypes.PUNCTUATION.SEMICOLON,
":": core_1.TokenTypes.PUNCTUATION.COLON,
".": core_1.TokenTypes.PUNCTUATION.OBJECT_ACCESSOR,
"?.": core_1.TokenTypes.PUNCTUATION.SAFE_OBJECT_ACCESSOR,
"\\": core_1.TokenTypes.PUNCTUATION.BACKSLASH,
"'": core_1.TokenTypes.PUNCTUATION.QUOTTES,
"\"": core_1.TokenTypes.PUNCTUATION.DOUBLE_QUOTTES,
"@": core_1.TokenTypes.PUNCTUATION.AT,
"?": core_1.TokenTypes.PUNCTUATION.EXMARK,
};
var Tokenizer = /** @class */ (function () {
function Tokenizer() {
}
/**
* Method to tokenize any String into basic tokens
* @param {string} str String to tokenize
* @param {number} [virtualLine] To tokenize a single line into a entire document, you can pass the line number to get all data correctly, including lines and colums
* @returns {Token[]} Return the token list from str
*/
Tokenizer.tokenize = function (str, virtualLine) {
var NUM_FORMAT = /[0-9]/;
var ID_FORMAT = /([a-zA-Z0-9À-ÿ]|_|–)/;
var tokens = [];
var lineNumber = 0;
var column = 0;
var onCommentBlock = false;
var onCommentLine = false;
var onText = false;
var aBracketsIndex = [];
var parentIndex = [];
var bracketIndex = [];
var sqBracketIndex = [];
var quottesIndex = [];
var commentBlockIndex = [];
var lastToken = (tokens.length > 0) ? tokens[tokens.length - 1] : undefined;
for (var charIndex = 0, len = str.length; charIndex < len; charIndex++) {
var fourChars = str.substring(charIndex, charIndex + 4);
var threeChars = str.substring(charIndex, charIndex + 3);
var twoChars = str.substring(charIndex, charIndex + 2);
var char = str.charAt(charIndex);
var token = void 0;
if (fourChars.length === 4 && symbolTokens[fourChars] && aBracketsIndex.length === 0) {
token = new core_1.Token(symbolTokens[fourChars], fourChars, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
charIndex += 3;
column += 4;
}
else if (threeChars.length === 3 && symbolTokens[threeChars] && aBracketsIndex.length === 0) {
token = new core_1.Token(symbolTokens[threeChars], threeChars, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
charIndex += 2;
column += 3;
}
else if (twoChars.length === 2 && symbolTokens[twoChars]) {
if (isLogicalOperator(symbolTokens[twoChars])) {
aBracketsIndex = [];
}
if (aBracketsIndex.length === 0) {
token = new core_1.Token(symbolTokens[twoChars], twoChars, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
charIndex += 1;
column += 2;
}
else if (symbolTokens[char]) {
token = new core_1.Token(symbolTokens[char], char, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
column++;
}
}
else if (symbolTokens[char]) {
token = new core_1.Token(symbolTokens[char], char, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
if (mustResetABracketIndex(token)) {
aBracketsIndex = [];
}
column++;
}
else if (NUM_FORMAT.test(char)) {
var numContent = '';
while (NUM_FORMAT.test(char) || char === '.' || char === ':' || char === '+' || char === '-' || char.toLowerCase() === 't' || char.toLowerCase() === 'z') {
numContent += char;
char = str.charAt(++charIndex);
}
if (numContent.indexOf(':') !== -1 && numContent.indexOf('-') !== -1) {
token = new core_1.Token(core_1.TokenTypes.LITERAL.DATETIME, numContent, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
}
else if (numContent.indexOf('-') !== -1) {
token = new core_1.Token(core_1.TokenTypes.LITERAL.DATE, numContent, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
}
else if (numContent.indexOf(':') !== -1) {
token = new core_1.Token(core_1.TokenTypes.LITERAL.TIME, numContent, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
}
else if (numContent.indexOf('.') !== -1) {
token = new core_1.Token(core_1.TokenTypes.LITERAL.DOUBLE, numContent, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
}
else {
token = new core_1.Token(core_1.TokenTypes.LITERAL.INTEGER, numContent, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
}
charIndex--;
column += numContent.length;
}
else if (ID_FORMAT.test(char)) {
var idContent = '';
while (ID_FORMAT.test(char)) {
idContent += char;
char = str.charAt(++charIndex);
}
charIndex--;
token = new core_1.Token(core_1.TokenTypes.IDENTIFIER, idContent, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
column += idContent.length;
}
else if (char === "\n") {
if (onCommentLine) {
onCommentLine = false;
}
lineNumber++;
column = 0;
}
else if (char !== "\t" && char !== " " && char.trim().length !== 0) {
token = new core_1.Token(core_1.TokenTypes.UNKNOWN, char, (virtualLine !== undefined && virtualLine >= 0) ? virtualLine : lineNumber, column);
column++;
}
else if (char === "\t") {
column += 4;
}
else {
column++;
}
if (token) {
if (!onText && !onCommentBlock && !onCommentLine && token.type === core_1.TokenTypes.PUNCTUATION.QUOTTES && (!lastToken || lastToken.text !== '\\')) {
token.type = core_1.TokenTypes.PUNCTUATION.QUOTTES_START;
onText = true;
token.parentToken = tokens.length - 1;
quottesIndex.push(tokens.length);
}
else if (onText && token.type === core_1.TokenTypes.PUNCTUATION.QUOTTES && (!lastToken || lastToken.text !== '\\')) {
token.type = core_1.TokenTypes.PUNCTUATION.QUOTTES_END;
onText = false;
if (quottesIndex.length > 0) {
var index = quottesIndex.pop();
if (index !== undefined) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (!onText && !onCommentBlock && !onCommentLine && (token.type === core_1.TokenTypes.COMMENT.BLOCK_START || token.type === core_1.TokenTypes.COMMENT.XML_START)) {
onCommentBlock = true;
token.parentToken = tokens.length - 1;
commentBlockIndex.push(tokens.length);
}
else if (onCommentBlock && (token.type === core_1.TokenTypes.COMMENT.BLOCK_END || token.type === core_1.TokenTypes.COMMENT.XML_END)) {
onCommentBlock = false;
if (commentBlockIndex.length > 0) {
var index = commentBlockIndex.pop();
if (index !== undefined) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (!onText && !onCommentBlock && !onCommentLine && (token.type === core_1.TokenTypes.COMMENT.LINE || token.type === core_1.TokenTypes.COMMENT.LINE_DOC)) {
onCommentLine = true;
}
else if (onText) {
token.type = core_1.TokenTypes.LITERAL.STRING;
}
else if (onCommentBlock || onCommentLine) {
token.type = core_1.TokenTypes.COMMENT.CONTENT;
}
else if (token.type === core_1.TokenTypes.OPERATOR.LOGICAL.LESS_THAN) {
aBracketsIndex.push(tokens.length);
}
else if (token.type === core_1.TokenTypes.OPERATOR.LOGICAL.GREATER_THAN) {
if (aBracketsIndex.length > 0) {
var index = aBracketsIndex.pop();
if (index !== undefined) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (token.type === core_1.TokenTypes.BRACKET.CURLY_OPEN && lastToken) {
bracketIndex.push(tokens.length);
}
else if (token.type === core_1.TokenTypes.BRACKET.CURLY_CLOSE) {
if (bracketIndex.length > 0) {
var index = bracketIndex.pop();
if (index !== undefined) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (token.type === core_1.TokenTypes.OPERATOR.PRIORITY.PARENTHESIS_OPEN) {
parentIndex.push(tokens.length);
}
else if (token.type === core_1.TokenTypes.OPERATOR.PRIORITY.PARENTHESIS_CLOSE) {
if (parentIndex.length > 0) {
var index = parentIndex.pop();
if (index !== undefined && tokens[index]) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (token.type === core_1.TokenTypes.BRACKET.SQUARE_OPEN) {
sqBracketIndex.push(tokens.length);
}
else if (token.type === core_1.TokenTypes.BRACKET.SQUARE_CLOSE) {
if (sqBracketIndex.length > 0) {
var index = sqBracketIndex.pop();
if (index !== undefined && tokens[index]) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (token.type === core_1.TokenTypes.BRACKET.SQUARE_OPEN) {
sqBracketIndex.push(tokens.length);
}
else if (token.type === core_1.TokenTypes.BRACKET.SQUARE_CLOSE) {
if (sqBracketIndex.length > 0) {
var index = sqBracketIndex.pop();
if (index !== undefined && tokens[index]) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
tokens.push(token);
}
}
return tokens;
};
return Tokenizer;
}());
exports.Tokenizer = Tokenizer;
function isLogicalOperator(symbol) {
return symbol === core_1.TokenTypes.OPERATOR.LOGICAL.INEQUALITY || symbol === core_1.TokenTypes.OPERATOR.LOGICAL.EQUALITY || symbol === core_1.TokenTypes.OPERATOR.LOGICAL.OR || symbol === core_1.TokenTypes.OPERATOR.LOGICAL.OR_ASSIGN || symbol === core_1.TokenTypes.OPERATOR.LOGICAL.AND || symbol === core_1.TokenTypes.OPERATOR.LOGICAL.AND_ASSIGN;
}
function mustResetABracketIndex(token) {
switch (token.type) {
case core_1.TokenTypes.PUNCTUATION.SEMICOLON:
case core_1.TokenTypes.BRACKET.CURLY_OPEN:
case core_1.TokenTypes.BRACKET.CURLY_CLOSE:
return true;
}
return false;
}
//# sourceMappingURL=tokenizer.js.map