mframejs
Version:
simple framework
317 lines (271 loc) • 11.8 kB
text/typescript
import { CharCodes } from './charcode';
// create charCode class we can use everytime
const charCode = new CharCodes();
/**
* Class for taking a expression and generating tokens for ast
*
*/
export class Tokenizer {
private baseTokenNo: number;
private token: any;
private expression: string;
private baseTokens: any[] = [];
private tokens: any[] = [];
private chars: any[] = [];
private curChar: any = null;
private curCharNo = 0;
private curtype: string = null;
private expressionLength: number;
private isMix: boolean; // if its a mix of text and interpolate expressions
private isOutsideExpression: boolean; // if mix stuff outside ${} is text only
private expressionOriginal: string;
constructor(expression: string) {
this.expressionOriginal = expression;
this.isMix = expression.indexOf('${') !== -1;
this.isMix = this.isMix ? true : expression.indexOf('@{') !== -1;
this.isOutsideExpression = this.isMix;
this.expression = this.setStrings(this.expressionOriginal);
this.expression = this.removeWhitespaceExpressions(this.expression);
this.curChar = this.expression.charCodeAt(this.curCharNo);
this.expressionLength = this.expression.length;
}
/**
* starts to parse and returns tokens
*
*/
public start(): any[] {
this.generateBaseTokens();
this.combineBaseTokens();
return this.tokens;
}
/**
* marks out text
*
*/
private setStrings(expression: string): string {
let text = this.isOutsideExpression;
let trimmed = '';
// if text add: "
if (text) {
trimmed = '"';
}
let count = 0;
for (let i = 0; i < expression.length; i++) {
switch (true) {
// if expressions start, add: " before
case text && expression.charCodeAt(i) === '$'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + '"' + expression[i];
text = false;
break;
case !text && expression.charCodeAt(i) === '$'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + expression[i];
break;
case text && expression.charCodeAt(i) === '@'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + '"' + expression[i];
text = false;
break;
case !text && expression.charCodeAt(i) === '@'.charCodeAt(0) && expression.charCodeAt(i + 1) === '{'.charCodeAt(0):
count++;
trimmed = trimmed + expression[i];
break;
// if expressions end, add: " before, if not end of string
case !text && expression.charCodeAt(i) === '}'.charCodeAt(0):
count--;
if (!count) {
trimmed = trimmed + expression[i] + '"';
text = true;
} else {
trimmed = trimmed + expression[i];
}
break;
// just add
default:
trimmed = trimmed + expression[i];
}
}
if (text) {
trimmed = trimmed + '"';
}
return trimmed;
}
/**
* trims down eveything thats not text so we dont need to check for whitespace later
*
*/
private removeWhitespaceExpressions(expression: string): string {
let text = false;
let trimmed = '';
let stringcharType: any;
for (let i = 0; i < expression.length; i++) {
switch (true) {
// turn on
case !text && charCode.STRING_START_END.has(expression.charCodeAt(i)):
trimmed = trimmed + expression[i];
stringcharType = expression.charCodeAt(i);
text = true;
break;
// turn of
case text && (stringcharType === expression.charCodeAt(i)):
trimmed = trimmed + expression[i];
text = false;
break;
// trim
case !text && charCode.WHITESPACE.has(expression.charCodeAt(i)):
// skip only
break;
// just add
default:
trimmed = trimmed + expression[i];
break;
}
}
return trimmed;
}
/**
* checks if we are at the end of expression
*
*/
private parsedAllChars(): boolean {
const done = this.curCharNo < this.expressionLength;
return !done;
}
/**
* goes to next char in expression
*
*/
private advanceChar(): void {
this.curCharNo++;
this.curChar = this.expression.charCodeAt(this.curCharNo);
}
/**
* adds token
*
*/
private addToken(): void {
const val = this.chars.map(a => String.fromCharCode(a)).join('');
this.baseTokens.push({
type: this.curtype,
value: this.curtype === 'number' ? parseFloat(val) : val
});
this.chars = [];
this.curtype = null;
}
/**
* loops expression and create base tokens
*
*/
private generateBaseTokens(): void {
let done = this.parsedAllChars();
while (!done) {
switch (true) {
case this.curtype === null && charCode.NUMBER.has(this.curChar):
this.curtype = 'number';
while (charCode.NUMBER.has(this.curChar) && !this.parsedAllChars()) {
this.chars.push(this.curChar);
this.advanceChar();
}
this.addToken();
break;
case this.curtype === null && charCode.STRING_START_END.has(this.curChar):
this.curtype = 'string';
const stringcharType = this.curChar;
this.advanceChar();
while ((stringcharType !== this.curChar) && !this.parsedAllChars()) {
this.chars.push(this.curChar);
this.advanceChar();
}
this.advanceChar();
this.addToken();
break;
case this.curtype === null && charCode.OPERATOR.has(this.curChar):
this.curtype = 'operator';
this.chars.push(this.curChar);
this.advanceChar();
this.addToken();
break;
default:
this.curtype = 'variable';
while (!charCode.OPERATOR.has(this.curChar) && !this.parsedAllChars()) {
this.chars.push(this.curChar);
this.advanceChar();
}
this.addToken();
}
done = this.parsedAllChars();
}
}
private advanceNextBaseToken() {
this.baseTokenNo++;
this.token = this.baseTokens[this.baseTokenNo];
}
private combineBaseTokens() {
this.tokens = [];
this.baseTokenNo = 0;
while (this.baseTokenNo < this.baseTokens.length) {
this.token = this.baseTokens[this.baseTokenNo];
switch (true) {
// VARIABLE
case this.token.type === 'variable':
const root = this.tokens[this.tokens.length - 1];
if (!root || root && root.value !== '.' && root.value !== '[') {
this.token.root = true;
}
this.tokens.push(this.token);
this.advanceNextBaseToken();
break;
// STRING
case this.token.type === 'string':
this.tokens.push(this.token);
this.advanceNextBaseToken();
break;
// OPERATOR
case this.token.type === 'operator':
const next1 = this.baseTokens[this.baseTokenNo + 1] ? this.baseTokens[this.baseTokenNo + 1] : undefined;
const next2 = this.baseTokens[this.baseTokenNo + 2] ? this.baseTokens[this.baseTokenNo + 2] : undefined;
if (next1 && next1.type === 'operator' && charCode.OPERATOR_COMBO.has(this.token.value + next1.value)) {
if (next2 && next2.type === 'operator' && charCode.OPERATOR_COMBO.has(this.token.value + next1.value + next2.value)) {
this.token.value = this.token.value + next1.value + next2.value;
this.tokens.push(this.token);
this.baseTokens.splice(this.baseTokenNo, 1);
this.baseTokens.splice(this.baseTokenNo, 1);
this.advanceNextBaseToken();
} else {
this.token.value = this.token.value + next1.value;
this.tokens.push(this.token);
this.baseTokens.splice(this.baseTokenNo, 1);
this.advanceNextBaseToken();
}
} else {
// check if operator and is minus, and if last was also optrator then set number
if ((this.token.value === '-' && next1.type === 'number')
&& this.tokens.length > 0 && this.tokens[this.tokens.length - 1].type === 'operator') {
this.token.type = 'number';
} else {
if ((this.token.value === '$' || this.token.value === '_') && next1.type === 'variable') {
next1.value = this.token.value + next1.value;
this.baseTokens.splice(this.baseTokenNo, 1);
} else {
this.tokens.push(this.token);
this.advanceNextBaseToken();
}
}
}
break;
// NUMBER
case this.token.type === 'number':
let check = this.baseTokens[this.baseTokenNo + 1] ? this.baseTokens[this.baseTokenNo + 1] : undefined;
while (check && check.type === 'number') {
this.token.value = this.token.value + check.value;
this.token.value = this.token.value * 1;
this.baseTokens.splice(this.baseTokenNo, 1);
check = this.baseTokens[this.baseTokenNo + 1] ? this.baseTokens[this.baseTokenNo + 1] : undefined;
}
this.tokens.push(this.token);
this.advanceNextBaseToken();
}
}
}
}