json-string-mapper
Version:
transform json string to key value pair map
353 lines (351 loc) • 13.3 kB
JavaScript
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import * as chars from './chars';
import { NumberWrapper } from '../facade/lang';
export var TokenType;
(function (TokenType) {
TokenType[TokenType["Character"] = 0] = "Character";
TokenType[TokenType["Identifier"] = 1] = "Identifier";
TokenType[TokenType["Keyword"] = 2] = "Keyword";
TokenType[TokenType["String"] = 3] = "String";
TokenType[TokenType["Operator"] = 4] = "Operator";
TokenType[TokenType["Number"] = 5] = "Number";
TokenType[TokenType["Error"] = 6] = "Error";
})(TokenType || (TokenType = {}));
var KEYWORDS = ['var', 'let', 'null', 'undefined', 'true', 'false', 'if', 'else', 'this'];
var Lexer = (function () {
function Lexer() {
}
Lexer.prototype.tokenize = function (text) {
var scanner = new _Scanner(text);
var tokens = [];
var token = scanner.scanToken();
while (token != null) {
tokens.push(token);
token = scanner.scanToken();
}
return tokens;
};
return Lexer;
}());
export { Lexer };
var Token = (function () {
function Token(index, type, numValue, strValue) {
this.index = index;
this.type = type;
this.numValue = numValue;
this.strValue = strValue;
}
Token.prototype.isCharacter = function (code) {
return this.type == TokenType.Character && this.numValue == code;
};
Token.prototype.isNumber = function () { return this.type == TokenType.Number; };
Token.prototype.isString = function () { return this.type == TokenType.String; };
Token.prototype.isOperator = function (operater) {
return this.type == TokenType.Operator && this.strValue == operater;
};
Token.prototype.isIdentifier = function () { return this.type == TokenType.Identifier; };
Token.prototype.isKeyword = function () { return this.type == TokenType.Keyword; };
Token.prototype.isKeywordLet = function () { return this.type == TokenType.Keyword && this.strValue == 'let'; };
Token.prototype.isKeywordNull = function () { return this.type == TokenType.Keyword && this.strValue == 'null'; };
Token.prototype.isKeywordUndefined = function () {
return this.type == TokenType.Keyword && this.strValue == 'undefined';
};
Token.prototype.isKeywordTrue = function () { return this.type == TokenType.Keyword && this.strValue == 'true'; };
Token.prototype.isKeywordFalse = function () { return this.type == TokenType.Keyword && this.strValue == 'false'; };
Token.prototype.isKeywordThis = function () { return this.type == TokenType.Keyword && this.strValue == 'this'; };
Token.prototype.isError = function () { return this.type == TokenType.Error; };
Token.prototype.toNumber = function () { return this.type == TokenType.Number ? this.numValue : -1; };
Token.prototype.toString = function () {
switch (this.type) {
case TokenType.Character:
case TokenType.Identifier:
case TokenType.Keyword:
case TokenType.Operator:
case TokenType.String:
case TokenType.Error:
return this.strValue;
case TokenType.Number:
return this.numValue.toString();
default:
return null;
}
};
return Token;
}());
export { Token };
function newCharacterToken(index, code) {
return new Token(index, TokenType.Character, code, String.fromCharCode(code));
}
function newIdentifierToken(index, text) {
return new Token(index, TokenType.Identifier, 0, text);
}
function newKeywordToken(index, text) {
return new Token(index, TokenType.Keyword, 0, text);
}
function newOperatorToken(index, text) {
return new Token(index, TokenType.Operator, 0, text);
}
function newStringToken(index, text) {
return new Token(index, TokenType.String, 0, text);
}
function newNumberToken(index, n) {
return new Token(index, TokenType.Number, n, '');
}
function newErrorToken(index, message) {
return new Token(index, TokenType.Error, 0, message);
}
export var EOF = new Token(-1, TokenType.Character, 0, '');
var _Scanner = (function () {
function _Scanner(input) {
this.input = input;
this.peek = 0;
this.index = -1;
this.length = input.length;
this.advance();
}
_Scanner.prototype.advance = function () {
this.peek = ++this.index >= this.length ? chars.$EOF : this.input.charCodeAt(this.index);
};
_Scanner.prototype.scanToken = function () {
var input = this.input, length = this.length;
var peek = this.peek, index = this.index;
// Skip whitespace.
while (peek <= chars.$SPACE) {
if (++index >= length) {
peek = chars.$EOF;
break;
}
else {
peek = input.charCodeAt(index);
}
}
this.peek = peek;
this.index = index;
if (index >= length) {
return null;
}
// Handle identifiers and numbers.
if (isIdentifierStart(peek))
return this.scanIdentifier();
if (chars.isDigit(peek))
return this.scanNumber(index);
var start = index;
switch (peek) {
case chars.$PERIOD:
this.advance();
return chars.isDigit(this.peek) ? this.scanNumber(start) :
newCharacterToken(start, chars.$PERIOD);
case chars.$LPAREN:
case chars.$RPAREN:
case chars.$LBRACE:
case chars.$RBRACE:
case chars.$LBRACKET:
case chars.$RBRACKET:
case chars.$COMMA:
case chars.$COLON:
case chars.$SEMICOLON:
return this.scanCharacter(start, peek);
case chars.$SQ:
case chars.$DQ:
return this.scanString();
case chars.$HASH:
case chars.$PLUS:
case chars.$MINUS:
case chars.$STAR:
case chars.$SLASH:
case chars.$PERCENT:
case chars.$CARET:
return this.scanOperator(start, String.fromCharCode(peek));
case chars.$QUESTION:
return this.scanComplexOperator(start, '?', chars.$PERIOD, '.');
case chars.$LT:
case chars.$GT:
return this.scanComplexOperator(start, String.fromCharCode(peek), chars.$EQ, '=');
case chars.$BANG:
case chars.$EQ:
return this.scanComplexOperator(start, String.fromCharCode(peek), chars.$EQ, '=', chars.$EQ, '=');
case chars.$AMPERSAND:
return this.scanComplexOperator(start, '&', chars.$AMPERSAND, '&');
case chars.$BAR:
return this.scanComplexOperator(start, '|', chars.$BAR, '|');
case chars.$NBSP:
while (chars.isWhitespace(this.peek))
this.advance();
return this.scanToken();
}
this.advance();
return this.error("Unexpected character [" + String.fromCharCode(peek) + "]", 0);
};
_Scanner.prototype.scanCharacter = function (start, code) {
this.advance();
return newCharacterToken(start, code);
};
_Scanner.prototype.scanOperator = function (start, str) {
this.advance();
return newOperatorToken(start, str);
};
/**
* Tokenize a 2/3 char long operator
*
* @param start start index in the expression
* @param one first symbol (always part of the operator)
* @param twoCode code point for the second symbol
* @param two second symbol (part of the operator when the second code point matches)
* @param threeCode code point for the third symbol
* @param three third symbol (part of the operator when provided and matches source expression)
* @returns {Token}
*/
_Scanner.prototype.scanComplexOperator = function (start, one, twoCode, two, threeCode, three) {
this.advance();
var str = one;
if (this.peek == twoCode) {
this.advance();
str += two;
}
if (threeCode != null && this.peek == threeCode) {
this.advance();
str += three;
}
return newOperatorToken(start, str);
};
_Scanner.prototype.scanIdentifier = function () {
var start = this.index;
this.advance();
while (isIdentifierPart(this.peek))
this.advance();
var str = this.input.substring(start, this.index);
return KEYWORDS.indexOf(str) > -1 ? newKeywordToken(start, str) :
newIdentifierToken(start, str);
};
_Scanner.prototype.scanNumber = function (start) {
var simple = (this.index === start);
this.advance(); // Skip initial digit.
while (true) {
if (chars.isDigit(this.peek)) {
// Do nothing.
}
else if (this.peek == chars.$PERIOD) {
simple = false;
}
else if (isExponentStart(this.peek)) {
this.advance();
if (isExponentSign(this.peek))
this.advance();
if (!chars.isDigit(this.peek))
return this.error('Invalid exponent', -1);
simple = false;
}
else {
break;
}
this.advance();
}
var str = this.input.substring(start, this.index);
var value = simple ? NumberWrapper.parseIntAutoRadix(str) : parseFloat(str);
return newNumberToken(start, value);
};
_Scanner.prototype.scanString = function () {
var start = this.index;
var quote = this.peek;
this.advance(); // Skip initial quote.
var buffer = '';
var marker = this.index;
var input = this.input;
while (this.peek != quote) {
if (this.peek == chars.$BACKSLASH) {
buffer += input.substring(marker, this.index);
this.advance();
var unescapedCode = void 0;
// Workaround for TS2.1-introduced type strictness
this.peek = this.peek;
if (this.peek == chars.$u) {
// 4 character hex code for unicode character.
var hex = input.substring(this.index + 1, this.index + 5);
if (/^[0-9a-f]+$/i.test(hex)) {
unescapedCode = parseInt(hex, 16);
}
else {
return this.error("Invalid unicode escape [\\u" + hex + "]", 0);
}
for (var i = 0; i < 5; i++) {
this.advance();
}
}
else {
unescapedCode = unescape(this.peek);
this.advance();
}
buffer += String.fromCharCode(unescapedCode);
marker = this.index;
}
else if (this.peek == chars.$EOF) {
return this.error('Unterminated quote', 0);
}
else {
this.advance();
}
}
var last = input.substring(marker, this.index);
this.advance(); // Skip terminating quote.
return newStringToken(start, buffer + last);
};
_Scanner.prototype.error = function (message, offset) {
var position = this.index + offset;
return newErrorToken(position, "Lexer Error: " + message + " at column " + position + " in expression [" + this.input + "]");
};
return _Scanner;
}());
function isIdentifierStart(code) {
return (chars.$a <= code && code <= chars.$z) || (chars.$A <= code && code <= chars.$Z) ||
(code == chars.$_) || (code == chars.$$);
}
export function isIdentifier(input) {
if (input.length == 0)
return false;
var scanner = new _Scanner(input);
if (!isIdentifierStart(scanner.peek))
return false;
scanner.advance();
while (scanner.peek !== chars.$EOF) {
if (!isIdentifierPart(scanner.peek))
return false;
scanner.advance();
}
return true;
}
function isIdentifierPart(code) {
return chars.isAsciiLetter(code) || chars.isDigit(code) || (code == chars.$_) ||
(code == chars.$$);
}
function isExponentStart(code) {
return code == chars.$e || code == chars.$E;
}
function isExponentSign(code) {
return code == chars.$MINUS || code == chars.$PLUS;
}
export function isQuote(code) {
return code === chars.$SQ || code === chars.$DQ || code === chars.$BT;
}
function unescape(code) {
switch (code) {
case chars.$n:
return chars.$LF;
case chars.$f:
return chars.$FF;
case chars.$r:
return chars.$CR;
case chars.$t:
return chars.$TAB;
case chars.$v:
return chars.$VTAB;
default:
return code;
}
}
//# sourceMappingURL=lexer.js.map