mframejs
Version:
simple framework
223 lines • 10.1 kB
JavaScript
import { CharCodes } from './charcode';
const charCode = new CharCodes();
export class Tokenizer {
constructor(expression) {
this.baseTokens = [];
this.tokens = [];
this.chars = [];
this.curChar = null;
this.curCharNo = 0;
this.curtype = null;
this.expressionOriginal = expression;
this.isMix = expression.indexOf('${') !== -1;
this.isMix = this.isMix ? true : expression.indexOf('@{') !== -1;
this.isOutsideExpression = this.isMix;
this.expression = this.setStrings(this.expressionOriginal);
this.expression = this.removeWhitespaceExpressions(this.expression);
this.curChar = this.expression.charCodeAt(this.curCharNo);
this.expressionLength = this.expression.length;
}
start() {
this.generateBaseTokens();
this.combineBaseTokens();
return this.tokens;
}
setStrings(expression) {
let text = this.isOutsideExpression;
let trimmed = '';
if (text) {
trimmed = '"';
}
let count = 0;
for (let i = 0; i < expression.length; i++) {
switch (true) {
case text && expression.charCodeAt(i) === '$'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + '"' + expression[i];
text = false;
break;
case !text && expression.charCodeAt(i) === '$'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + expression[i];
break;
case text && expression.charCodeAt(i) === '@'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + '"' + expression[i];
text = false;
break;
case !text && expression.charCodeAt(i) === '@'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + expression[i];
break;
case !text && expression.charCodeAt(i) === '}'.charCodeAt(0):
count--;
if (!count) {
trimmed = trimmed + expression[i] + '"';
text = true;
}
else {
trimmed = trimmed + expression[i];
}
break;
default:
trimmed = trimmed + expression[i];
}
}
if (text) {
trimmed = trimmed + '"';
}
return trimmed;
}
removeWhitespaceExpressions(expression) {
let text = false;
let trimmed = '';
let stringcharType;
for (let i = 0; i < expression.length; i++) {
switch (true) {
case !text && charCode.STRING_START_END.has(expression.charCodeAt(i)):
trimmed = trimmed + expression[i];
stringcharType = expression.charCodeAt(i);
text = true;
break;
case text && (stringcharType === expression.charCodeAt(i)):
trimmed = trimmed + expression[i];
text = false;
break;
case !text && charCode.WHITESPACE.has(expression.charCodeAt(i)):
break;
default:
trimmed = trimmed + expression[i];
break;
}
}
return trimmed;
}
parsedAllChars() {
const done = this.curCharNo < this.expressionLength;
return !done;
}
advanceChar() {
this.curCharNo++;
this.curChar = this.expression.charCodeAt(this.curCharNo);
}
addToken() {
const val = this.chars.map(a => String.fromCharCode(a)).join('');
this.baseTokens.push({
type: this.curtype,
value: this.curtype === 'number' ? parseFloat(val) : val
});
this.chars = [];
this.curtype = null;
}
generateBaseTokens() {
let done = this.parsedAllChars();
while (!done) {
switch (true) {
case this.curtype === null && charCode.NUMBER.has(this.curChar):
this.curtype = 'number';
while (charCode.NUMBER.has(this.curChar) && !this.parsedAllChars()) {
this.chars.push(this.curChar);
this.advanceChar();
}
this.addToken();
break;
case this.curtype === null && charCode.STRING_START_END.has(this.curChar):
this.curtype = 'string';
const stringcharType = this.curChar;
this.advanceChar();
while ((stringcharType !== this.curChar) && !this.parsedAllChars()) {
this.chars.push(this.curChar);
this.advanceChar();
}
this.advanceChar();
this.addToken();
break;
case this.curtype === null && charCode.OPERATOR.has(this.curChar):
this.curtype = 'operator';
this.chars.push(this.curChar);
this.advanceChar();
this.addToken();
break;
default:
this.curtype = 'variable';
while (!charCode.OPERATOR.has(this.curChar) && !this.parsedAllChars()) {
this.chars.push(this.curChar);
this.advanceChar();
}
this.addToken();
}
done = this.parsedAllChars();
}
}
advanceNextBaseToken() {
this.baseTokenNo++;
this.token = this.baseTokens[this.baseTokenNo];
}
combineBaseTokens() {
this.tokens = [];
this.baseTokenNo = 0;
while (this.baseTokenNo < this.baseTokens.length) {
this.token = this.baseTokens[this.baseTokenNo];
switch (true) {
case this.token.type === 'variable':
const root = this.tokens[this.tokens.length - 1];
if (!root || root && root.value !== '.' && root.value !== '[') {
this.token.root = true;
}
this.tokens.push(this.token);
this.advanceNextBaseToken();
break;
case this.token.type === 'string':
this.tokens.push(this.token);
this.advanceNextBaseToken();
break;
case this.token.type === 'operator':
const next1 = this.baseTokens[this.baseTokenNo + 1] ? this.baseTokens[this.baseTokenNo + 1] : undefined;
const next2 = this.baseTokens[this.baseTokenNo + 2] ? this.baseTokens[this.baseTokenNo + 2] : undefined;
if (next1 && next1.type === 'operator' && charCode.OPERATOR_COMBO.has(this.token.value + next1.value)) {
if (next2 && next2.type === 'operator' && charCode.OPERATOR_COMBO.has(this.token.value + next1.value + next2.value)) {
this.token.value = this.token.value + next1.value + next2.value;
this.tokens.push(this.token);
this.baseTokens.splice(this.baseTokenNo, 1);
this.baseTokens.splice(this.baseTokenNo, 1);
this.advanceNextBaseToken();
}
else {
this.token.value = this.token.value + next1.value;
this.tokens.push(this.token);
this.baseTokens.splice(this.baseTokenNo, 1);
this.advanceNextBaseToken();
}
}
else {
if ((this.token.value === '-' && next1.type === 'number')
&& this.tokens.length > 0 && this.tokens[this.tokens.length - 1].type === 'operator') {
this.token.type = 'number';
}
else {
if ((this.token.value === '$' || this.token.value === '_') && next1.type === 'variable') {
next1.value = this.token.value + next1.value;
this.baseTokens.splice(this.baseTokenNo, 1);
}
else {
this.tokens.push(this.token);
this.advanceNextBaseToken();
}
}
}
break;
case this.token.type === 'number':
let check = this.baseTokens[this.baseTokenNo + 1] ? this.baseTokens[this.baseTokenNo + 1] : undefined;
while (check && check.type === 'number') {
this.token.value = this.token.value + check.value;
this.token.value = this.token.value * 1;
this.baseTokens.splice(this.baseTokenNo, 1);
check = this.baseTokens[this.baseTokenNo + 1] ? this.baseTokens[this.baseTokenNo + 1] : undefined;
}
this.tokens.push(this.token);
this.advanceNextBaseToken();
}
}
}
}
//# sourceMappingURL=tokenizer.js.map