@aurahelper/languages
Version:
Language Libraries to work with XML, Aura, Apex... files. tokenizers, parsers, system classes and much more
326 lines • 18.7 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AuraTokenizer = void 0;
var core_1 = require("@aurahelper/core");
var StrUtils = core_1.CoreUtils.StrUtils;
var Utils = core_1.CoreUtils.Utils;
var Validator = core_1.CoreUtils.Validator;
var symbolTokens = {
'{': core_1.AuraTokenTypes.BRACKET.CURLY_OPEN,
'}': core_1.AuraTokenTypes.BRACKET.CURLY_CLOSE,
'<': core_1.AuraTokenTypes.BRACKET.START_TAG_OPEN,
'>': core_1.AuraTokenTypes.BRACKET.START_TAG_CLOSE,
'</': core_1.AuraTokenTypes.BRACKET.END_TAG_OPEN,
'/>': core_1.AuraTokenTypes.BRACKET.TAG_EMPTY_CLOSE,
'<? ': core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_OPEN,
'?>': core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_CLOSE,
'<!': core_1.AuraTokenTypes.BRACKET.TAG_ENTITY_OPEN,
'"': core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES,
'<!--': core_1.AuraTokenTypes.COMMENT.START,
'-->': core_1.AuraTokenTypes.COMMENT.END,
',': core_1.AuraTokenTypes.PUNCTUATION.COMMA,
':': core_1.AuraTokenTypes.PUNCTUATION.COLON,
';': core_1.AuraTokenTypes.PUNCTUATION.SEMICOLON,
'\\': core_1.AuraTokenTypes.PUNCTUATION.BACKSLASH,
'@': core_1.AuraTokenTypes.PUNCTUATION.AT,
'?': core_1.AuraTokenTypes.PUNCTUATION.EXMARK,
'=': core_1.AuraTokenTypes.OPERATOR.ASSIGN.ASSIGN,
".": core_1.AuraTokenTypes.PUNCTUATION.OBJECT_ACCESSOR,
"?.": core_1.AuraTokenTypes.PUNCTUATION.SAFE_OBJECT_ACCESSOR,
};
/**
* Class to tokenize Aura XML files
*/
var AuraTokenizer = /** @class */ (function () {
function AuraTokenizer() {
}
/**
* Method to Tokenize Aura XML files
* @param {string} filePathOrContent File path or File content to tokenize
* @param {number} [tabSize] File tab size
* @returns {Token[]} Return file tokens
*/
AuraTokenizer.tokenize = function (filePathOrContent, tabSize) {
if (!tabSize) {
tabSize = 4;
}
var content;
if (Utils.isString(filePathOrContent)) {
try {
content = core_1.FileReader.readFileSync(Validator.validateFilePath(filePathOrContent));
}
catch (error) {
content = filePathOrContent;
}
}
else {
throw new Error('You must to select a file path,or file content');
}
var NUM_FORMAT = /[0-9]/;
var ID_FORMAT = /([a-zA-Z0-9À-ÿ]|_|–)/;
content = StrUtils.replace(content, '\r\n', '\n');
var tokens = [];
var lineNumber = 0;
var column = 0;
var onCommentBlock = false;
var onText = false;
var startTagIndex = [];
var quottesIndex = [];
var commentBlockIndex = [];
for (var charIndex = 0, len = content.length; charIndex < len; charIndex++) {
var fourChars = content.substring(charIndex, charIndex + 4);
var threeChars = content.substring(charIndex, charIndex + 3);
var twoChars = content.substring(charIndex, charIndex + 2);
var char = content.charAt(charIndex);
var token = void 0;
var lastToken = (tokens.length > 0) ? tokens[tokens.length - 1] : undefined;
var twoLastToken = (tokens.length > 1) ? tokens[tokens.length - 2] : undefined;
if (fourChars.length === 4 && symbolTokens[fourChars]) {
token = new core_1.Token(symbolTokens[fourChars], fourChars, lineNumber, column);
charIndex += 3;
column += 4;
}
else if (threeChars.length === 3 && symbolTokens[threeChars]) {
token = new core_1.Token(symbolTokens[threeChars], threeChars, lineNumber, column);
charIndex += 2;
column += 3;
}
else if (twoChars.length === 2 && symbolTokens[twoChars]) {
token = new core_1.Token(symbolTokens[twoChars], twoChars, lineNumber, column);
charIndex += 1;
column += 2;
}
else if (symbolTokens[char]) {
token = new core_1.Token(symbolTokens[char], char, lineNumber, column);
column++;
}
else if (NUM_FORMAT.test(char)) {
var numContent = '';
while (NUM_FORMAT.test(char) || char === '.' || char === ':' || char === '+' || char === '-' || char.toLowerCase() === 't' || char.toLowerCase() === 'z') {
numContent += char;
char = content.charAt(++charIndex);
}
if (numContent.indexOf(':') !== -1 && numContent.indexOf('-') !== -1) {
token = new core_1.Token(core_1.AuraTokenTypes.LITERAL.DATETIME, numContent, lineNumber, column);
}
else if (numContent.indexOf('-') !== -1) {
token = new core_1.Token(core_1.AuraTokenTypes.LITERAL.DATE, numContent, lineNumber, column);
}
else if (numContent.indexOf(':') !== -1) {
token = new core_1.Token(core_1.AuraTokenTypes.LITERAL.TIME, numContent, lineNumber, column);
}
else if (numContent.indexOf('.') !== -1) {
token = new core_1.Token(core_1.AuraTokenTypes.LITERAL.DOUBLE, numContent, lineNumber, column);
}
else {
token = new core_1.Token(core_1.AuraTokenTypes.LITERAL.INTEGER, numContent, lineNumber, column);
}
charIndex--;
column += numContent.length;
}
else if (ID_FORMAT.test(char)) {
var idContent = '';
while (ID_FORMAT.test(char)) {
idContent += char;
char = content.charAt(++charIndex);
}
charIndex--;
token = new core_1.Token(core_1.AuraTokenTypes.IDENTIFIER, idContent, lineNumber, column);
column += idContent.length;
}
else if (char === "\n") {
lineNumber++;
column = 0;
}
else if (char !== "\t" && char !== " " && char.trim().length !== 0) {
token = new core_1.Token(core_1.AuraTokenTypes.UNKNOWN, char, lineNumber, column);
column++;
}
else if (char === "\t") {
column += tabSize;
}
else {
column++;
}
if (token !== undefined) {
if (!onText && !onCommentBlock && token.type === core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES && (!lastToken || lastToken.text !== '\\')) {
token.type = core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES_START;
onText = true;
token.parentToken = tokens.length - 1;
quottesIndex.push(tokens.length);
}
else if (onText && token.type === core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES && (!lastToken || lastToken.text !== '\\')) {
if (twoLastToken && twoLastToken.type === core_1.AuraTokenTypes.OPERATOR.ASSIGN.ASSIGN && lastToken && lastToken.type === core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES_START) {
if (startTagIndex.length > 0) {
var index = startTagIndex[startTagIndex.length - 1];
var tagToken = tokens[index];
if (tagToken.type === core_1.AuraTokenTypes.BRACKET.START_TAG_OPEN || tagToken.type === core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_OPEN) {
tokens.push(new core_1.Token(core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE_VALUE, '', lastToken.range.end.line, lastToken.range.end.character));
}
}
}
token.type = core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES_END;
onText = false;
if (quottesIndex.length > 0) {
var index = quottesIndex.pop();
if (index !== undefined) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
token.parentToken = tokens[index].parentToken;
}
}
}
else if (!onText && !onCommentBlock && token.type === core_1.AuraTokenTypes.COMMENT.START) {
onCommentBlock = true;
token.parentToken = tokens.length - 1;
commentBlockIndex.push(tokens.length);
}
else if (onCommentBlock && token.type === core_1.AuraTokenTypes.COMMENT.END) {
onCommentBlock = false;
if (commentBlockIndex.length > 0) {
var index = commentBlockIndex.pop();
if (index !== undefined) {
token.pairToken = index;
tokens[index].pairToken = tokens.length;
token.parentToken = tokens[index].parentToken;
}
}
}
else if (onText) {
if (twoLastToken && twoLastToken.type === core_1.AuraTokenTypes.OPERATOR.ASSIGN.ASSIGN && lastToken && lastToken.type === core_1.AuraTokenTypes.PUNCTUATION.DOUBLE_QUOTTES_START) {
if (startTagIndex.length > 0) {
var index = startTagIndex[startTagIndex.length - 1];
var tagToken = tokens[index];
if (tagToken.type === core_1.AuraTokenTypes.BRACKET.START_TAG_OPEN || tagToken.type === core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_OPEN) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE_VALUE;
token.parentToken = lastToken.parentToken;
}
}
}
else if (lastToken && lastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE_VALUE) {
var whitespaces = (lastToken.range.start.line !== token.range.start.line) ? 0 : token.range.start.character - lastToken.range.end.character;
token = new core_1.Token(core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE_VALUE, lastToken.text + getWhitespaces(whitespaces) + token.text, lastToken.range.start.line, lastToken.range.start.character);
token.parentToken = lastToken.parentToken;
tokens.pop();
lastToken = tokens[tokens.length - 1];
quottesIndex.pop();
}
else {
token.type = core_1.AuraTokenTypes.LITERAL.STRING;
if (quottesIndex.length > 0) {
token.parentToken = quottesIndex[quottesIndex.length - 1];
}
}
}
else if (onCommentBlock) {
token.type = core_1.AuraTokenTypes.COMMENT.CONTENT;
if (commentBlockIndex.length > 0) {
token.parentToken = commentBlockIndex[commentBlockIndex.length - 1];
}
}
else if (token.type === core_1.AuraTokenTypes.OPERATOR.ASSIGN.ASSIGN && lastToken && lastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE) {
token.parentToken = tokens.length - 1;
}
else if (token.type === core_1.AuraTokenTypes.BRACKET.START_TAG_OPEN || token.type === core_1.AuraTokenTypes.BRACKET.END_TAG_OPEN || token.type === core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_OPEN || token.type === core_1.AuraTokenTypes.BRACKET.TAG_ENTITY_OPEN) {
if (startTagIndex.length > 0) {
var index = startTagIndex.pop();
if (index !== undefined) {
tokens[index].type = core_1.AuraTokenTypes.BRACKET.INCOMPLETE_TAG_OPEN;
}
}
else {
startTagIndex.push(tokens.length);
}
}
else if (token.type === core_1.AuraTokenTypes.BRACKET.START_TAG_CLOSE || token.type === core_1.AuraTokenTypes.BRACKET.TAG_EMPTY_CLOSE || token.type === core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_CLOSE) {
if (startTagIndex.length > 0) {
var index = startTagIndex.pop();
if (index !== undefined) {
var pairToken = tokens[index];
if (token.type === core_1.AuraTokenTypes.BRACKET.TAG_EMPTY_CLOSE) {
tokens[index].type = core_1.AuraTokenTypes.BRACKET.TAG_EMPTY_OPEN;
}
else if (token.type === core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_CLOSE) {
tokens[index].type = core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_OPEN;
}
else if (pairToken.type === core_1.AuraTokenTypes.BRACKET.TAG_ENTITY_OPEN) {
token.type = core_1.AuraTokenTypes.BRACKET.TAG_ENTITY_CLOSE;
}
else if (pairToken.type === core_1.AuraTokenTypes.BRACKET.END_TAG_OPEN) {
token.type = core_1.AuraTokenTypes.BRACKET.END_TAG_CLOSE;
}
else if (pairToken.type === core_1.AuraTokenTypes.BRACKET.START_TAG_OPEN) {
token.type = core_1.AuraTokenTypes.BRACKET.START_TAG_CLOSE;
}
token.pairToken = index;
tokens[index].pairToken = tokens.length;
}
}
}
else if (token.type === core_1.AuraTokenTypes.IDENTIFIER) {
if (lastToken) {
if (startTagIndex.length > 0) {
var index = startTagIndex[startTagIndex.length - 1];
if (lastToken.type === core_1.AuraTokenTypes.BRACKET.START_TAG_OPEN || lastToken.type === core_1.AuraTokenTypes.BRACKET.TAG_EXMARK_OPEN) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.NAME;
token.parentToken = index;
}
else if (lastToken.type === core_1.AuraTokenTypes.BRACKET.END_TAG_OPEN) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.NAME;
token.parentToken = index;
}
else if (lastToken.type === core_1.AuraTokenTypes.BRACKET.TAG_ENTITY_OPEN) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.NAME;
token.parentToken = index;
}
else if (lastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.NAME || (twoLastToken && twoLastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE_VALUE)) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE;
token.parentToken = index;
}
else if (twoLastToken && twoLastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE && lastToken.type === core_1.AuraTokenTypes.PUNCTUATION.COLON) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.NAME;
token.parentToken = index;
tokens.pop();
tokens.pop();
token = new core_1.Token(core_1.AuraTokenTypes.ENTITY.TAG.ATTRIBUTE, twoLastToken.text + lastToken.text + token.text, twoLastToken.range.start.line, twoLastToken.range.start.character);
lastToken = tokens[tokens.length - 1];
}
else if (lastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.NAME && token.type === core_1.AuraTokenTypes.PUNCTUATION.COLON) {
tokens[tokens.length - 1].type = core_1.AuraTokenTypes.ENTITY.NAMESPACE;
token.parentToken = index;
}
else if (twoLastToken && twoLastToken.type === core_1.AuraTokenTypes.ENTITY.TAG.NAME && lastToken.type === core_1.AuraTokenTypes.PUNCTUATION.COLON) {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.NAME;
token.parentToken = index;
tokens.pop();
tokens.pop();
token = new core_1.Token(core_1.AuraTokenTypes.ENTITY.TAG.NAME, twoLastToken.text + lastToken.text + token.text, twoLastToken.range.start.line, twoLastToken.range.start.character);
lastToken = tokens[tokens.length - 1];
}
else {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.CONTENT;
token.parentToken = index;
}
}
else {
token.type = core_1.AuraTokenTypes.ENTITY.TAG.CONTENT;
token.parentToken = lastToken.parentToken;
}
}
}
tokens.push(token);
}
}
return tokens;
};
return AuraTokenizer;
}());
exports.AuraTokenizer = AuraTokenizer;
function getWhitespaces(number) {
var ws = '';
for (var index = 0; index < number; index++) {
ws += ' ';
}
return ws;
}
//# sourceMappingURL=tokenizer.js.map