@angular/compiler
Version:
Angular - the compiler library
951 lines • 139 kB
JavaScript
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
(function (factory) {
if (typeof module === "object" && typeof module.exports === "object") {
var v = factory(require, exports);
if (v !== undefined) module.exports = v;
}
else if (typeof define === "function" && define.amd) {
define("@angular/compiler/src/ml_parser/lexer", ["require", "exports", "tslib", "@angular/compiler/src/chars", "@angular/compiler/src/parse_util", "@angular/compiler/src/ml_parser/interpolation_config", "@angular/compiler/src/ml_parser/tags"], factory);
}
})(function (require, exports) {
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.CursorError = exports.tokenize = exports.TokenizeResult = exports.TokenError = exports.Token = exports.TokenType = void 0;
var tslib_1 = require("tslib");
var chars = require("@angular/compiler/src/chars");
var parse_util_1 = require("@angular/compiler/src/parse_util");
var interpolation_config_1 = require("@angular/compiler/src/ml_parser/interpolation_config");
var tags_1 = require("@angular/compiler/src/ml_parser/tags");
var TokenType;
(function (TokenType) {
TokenType[TokenType["TAG_OPEN_START"] = 0] = "TAG_OPEN_START";
TokenType[TokenType["TAG_OPEN_END"] = 1] = "TAG_OPEN_END";
TokenType[TokenType["TAG_OPEN_END_VOID"] = 2] = "TAG_OPEN_END_VOID";
TokenType[TokenType["TAG_CLOSE"] = 3] = "TAG_CLOSE";
TokenType[TokenType["INCOMPLETE_TAG_OPEN"] = 4] = "INCOMPLETE_TAG_OPEN";
TokenType[TokenType["TEXT"] = 5] = "TEXT";
TokenType[TokenType["ESCAPABLE_RAW_TEXT"] = 6] = "ESCAPABLE_RAW_TEXT";
TokenType[TokenType["RAW_TEXT"] = 7] = "RAW_TEXT";
TokenType[TokenType["COMMENT_START"] = 8] = "COMMENT_START";
TokenType[TokenType["COMMENT_END"] = 9] = "COMMENT_END";
TokenType[TokenType["CDATA_START"] = 10] = "CDATA_START";
TokenType[TokenType["CDATA_END"] = 11] = "CDATA_END";
TokenType[TokenType["ATTR_NAME"] = 12] = "ATTR_NAME";
TokenType[TokenType["ATTR_QUOTE"] = 13] = "ATTR_QUOTE";
TokenType[TokenType["ATTR_VALUE"] = 14] = "ATTR_VALUE";
TokenType[TokenType["DOC_TYPE"] = 15] = "DOC_TYPE";
TokenType[TokenType["EXPANSION_FORM_START"] = 16] = "EXPANSION_FORM_START";
TokenType[TokenType["EXPANSION_CASE_VALUE"] = 17] = "EXPANSION_CASE_VALUE";
TokenType[TokenType["EXPANSION_CASE_EXP_START"] = 18] = "EXPANSION_CASE_EXP_START";
TokenType[TokenType["EXPANSION_CASE_EXP_END"] = 19] = "EXPANSION_CASE_EXP_END";
TokenType[TokenType["EXPANSION_FORM_END"] = 20] = "EXPANSION_FORM_END";
TokenType[TokenType["EOF"] = 21] = "EOF";
})(TokenType = exports.TokenType || (exports.TokenType = {}));
var Token = /** @class */ (function () {
function Token(type, parts, sourceSpan) {
this.type = type;
this.parts = parts;
this.sourceSpan = sourceSpan;
}
return Token;
}());
exports.Token = Token;
var TokenError = /** @class */ (function (_super) {
tslib_1.__extends(TokenError, _super);
function TokenError(errorMsg, tokenType, span) {
var _this = _super.call(this, span, errorMsg) || this;
_this.tokenType = tokenType;
return _this;
}
return TokenError;
}(parse_util_1.ParseError));
exports.TokenError = TokenError;
var TokenizeResult = /** @class */ (function () {
function TokenizeResult(tokens, errors, nonNormalizedIcuExpressions) {
this.tokens = tokens;
this.errors = errors;
this.nonNormalizedIcuExpressions = nonNormalizedIcuExpressions;
}
return TokenizeResult;
}());
exports.TokenizeResult = TokenizeResult;
function tokenize(source, url, getTagDefinition, options) {
if (options === void 0) { options = {}; }
var tokenizer = new _Tokenizer(new parse_util_1.ParseSourceFile(source, url), getTagDefinition, options);
tokenizer.tokenize();
return new TokenizeResult(mergeTextTokens(tokenizer.tokens), tokenizer.errors, tokenizer.nonNormalizedIcuExpressions);
}
exports.tokenize = tokenize;
var _CR_OR_CRLF_REGEXP = /\r\n?/g;
function _unexpectedCharacterErrorMsg(charCode) {
var char = charCode === chars.$EOF ? 'EOF' : String.fromCharCode(charCode);
return "Unexpected character \"" + char + "\"";
}
function _unknownEntityErrorMsg(entitySrc) {
return "Unknown entity \"" + entitySrc + "\" - use the \"&#<decimal>;\" or \"&#x<hex>;\" syntax";
}
function _unparsableEntityErrorMsg(type, entityStr) {
return "Unable to parse entity \"" + entityStr + "\" - " + type + " character reference entities must end with \";\"";
}
var CharacterReferenceType;
(function (CharacterReferenceType) {
CharacterReferenceType["HEX"] = "hexadecimal";
CharacterReferenceType["DEC"] = "decimal";
})(CharacterReferenceType || (CharacterReferenceType = {}));
var _ControlFlowError = /** @class */ (function () {
function _ControlFlowError(error) {
this.error = error;
}
return _ControlFlowError;
}());
// See http://www.w3.org/TR/html51/syntax.html#writing
var _Tokenizer = /** @class */ (function () {
/**
* @param _file The html source file being tokenized.
* @param _getTagDefinition A function that will retrieve a tag definition for a given tag name.
* @param options Configuration of the tokenization.
*/
function _Tokenizer(_file, _getTagDefinition, options) {
this._getTagDefinition = _getTagDefinition;
this._currentTokenStart = null;
this._currentTokenType = null;
this._expansionCaseStack = [];
this._inInterpolation = false;
this.tokens = [];
this.errors = [];
this.nonNormalizedIcuExpressions = [];
this._tokenizeIcu = options.tokenizeExpansionForms || false;
this._interpolationConfig = options.interpolationConfig || interpolation_config_1.DEFAULT_INTERPOLATION_CONFIG;
this._leadingTriviaCodePoints =
options.leadingTriviaChars && options.leadingTriviaChars.map(function (c) { return c.codePointAt(0) || 0; });
var range = options.range || { endPos: _file.content.length, startPos: 0, startLine: 0, startCol: 0 };
this._cursor = options.escapedString ? new EscapedCharacterCursor(_file, range) :
new PlainCharacterCursor(_file, range);
this._preserveLineEndings = options.preserveLineEndings || false;
this._escapedString = options.escapedString || false;
this._i18nNormalizeLineEndingsInICUs = options.i18nNormalizeLineEndingsInICUs || false;
try {
this._cursor.init();
}
catch (e) {
this.handleError(e);
}
}
_Tokenizer.prototype._processCarriageReturns = function (content) {
if (this._preserveLineEndings) {
return content;
}
// http://www.w3.org/TR/html5/syntax.html#preprocessing-the-input-stream
// In order to keep the original position in the source, we can not
// pre-process it.
// Instead CRs are processed right before instantiating the tokens.
return content.replace(_CR_OR_CRLF_REGEXP, '\n');
};
_Tokenizer.prototype.tokenize = function () {
while (this._cursor.peek() !== chars.$EOF) {
var start = this._cursor.clone();
try {
if (this._attemptCharCode(chars.$LT)) {
if (this._attemptCharCode(chars.$BANG)) {
if (this._attemptCharCode(chars.$LBRACKET)) {
this._consumeCdata(start);
}
else if (this._attemptCharCode(chars.$MINUS)) {
this._consumeComment(start);
}
else {
this._consumeDocType(start);
}
}
else if (this._attemptCharCode(chars.$SLASH)) {
this._consumeTagClose(start);
}
else {
this._consumeTagOpen(start);
}
}
else if (!(this._tokenizeIcu && this._tokenizeExpansionForm())) {
this._consumeText();
}
}
catch (e) {
this.handleError(e);
}
}
this._beginToken(TokenType.EOF);
this._endToken([]);
};
/**
* @returns whether an ICU token has been created
* @internal
*/
_Tokenizer.prototype._tokenizeExpansionForm = function () {
if (this.isExpansionFormStart()) {
this._consumeExpansionFormStart();
return true;
}
if (isExpansionCaseStart(this._cursor.peek()) && this._isInExpansionForm()) {
this._consumeExpansionCaseStart();
return true;
}
if (this._cursor.peek() === chars.$RBRACE) {
if (this._isInExpansionCase()) {
this._consumeExpansionCaseEnd();
return true;
}
if (this._isInExpansionForm()) {
this._consumeExpansionFormEnd();
return true;
}
}
return false;
};
_Tokenizer.prototype._beginToken = function (type, start) {
if (start === void 0) { start = this._cursor.clone(); }
this._currentTokenStart = start;
this._currentTokenType = type;
};
_Tokenizer.prototype._endToken = function (parts, end) {
if (this._currentTokenStart === null) {
throw new TokenError('Programming error - attempted to end a token when there was no start to the token', this._currentTokenType, this._cursor.getSpan(end));
}
if (this._currentTokenType === null) {
throw new TokenError('Programming error - attempted to end a token which has no token type', null, this._cursor.getSpan(this._currentTokenStart));
}
var token = new Token(this._currentTokenType, parts, this._cursor.getSpan(this._currentTokenStart, this._leadingTriviaCodePoints));
this.tokens.push(token);
this._currentTokenStart = null;
this._currentTokenType = null;
return token;
};
_Tokenizer.prototype._createError = function (msg, span) {
if (this._isInExpansionForm()) {
msg += " (Do you have an unescaped \"{\" in your template? Use \"{{ '{' }}\") to escape it.)";
}
var error = new TokenError(msg, this._currentTokenType, span);
this._currentTokenStart = null;
this._currentTokenType = null;
return new _ControlFlowError(error);
};
_Tokenizer.prototype.handleError = function (e) {
if (e instanceof CursorError) {
e = this._createError(e.msg, this._cursor.getSpan(e.cursor));
}
if (e instanceof _ControlFlowError) {
this.errors.push(e.error);
}
else {
throw e;
}
};
_Tokenizer.prototype._attemptCharCode = function (charCode) {
if (this._cursor.peek() === charCode) {
this._cursor.advance();
return true;
}
return false;
};
_Tokenizer.prototype._attemptCharCodeCaseInsensitive = function (charCode) {
if (compareCharCodeCaseInsensitive(this._cursor.peek(), charCode)) {
this._cursor.advance();
return true;
}
return false;
};
_Tokenizer.prototype._requireCharCode = function (charCode) {
var location = this._cursor.clone();
if (!this._attemptCharCode(charCode)) {
throw this._createError(_unexpectedCharacterErrorMsg(this._cursor.peek()), this._cursor.getSpan(location));
}
};
_Tokenizer.prototype._attemptStr = function (chars) {
var len = chars.length;
if (this._cursor.charsLeft() < len) {
return false;
}
var initialPosition = this._cursor.clone();
for (var i = 0; i < len; i++) {
if (!this._attemptCharCode(chars.charCodeAt(i))) {
// If attempting to parse the string fails, we want to reset the parser
// to where it was before the attempt
this._cursor = initialPosition;
return false;
}
}
return true;
};
_Tokenizer.prototype._attemptStrCaseInsensitive = function (chars) {
for (var i = 0; i < chars.length; i++) {
if (!this._attemptCharCodeCaseInsensitive(chars.charCodeAt(i))) {
return false;
}
}
return true;
};
_Tokenizer.prototype._requireStr = function (chars) {
var location = this._cursor.clone();
if (!this._attemptStr(chars)) {
throw this._createError(_unexpectedCharacterErrorMsg(this._cursor.peek()), this._cursor.getSpan(location));
}
};
_Tokenizer.prototype._attemptCharCodeUntilFn = function (predicate) {
while (!predicate(this._cursor.peek())) {
this._cursor.advance();
}
};
_Tokenizer.prototype._requireCharCodeUntilFn = function (predicate, len) {
var start = this._cursor.clone();
this._attemptCharCodeUntilFn(predicate);
if (this._cursor.diff(start) < len) {
throw this._createError(_unexpectedCharacterErrorMsg(this._cursor.peek()), this._cursor.getSpan(start));
}
};
_Tokenizer.prototype._attemptUntilChar = function (char) {
while (this._cursor.peek() !== char) {
this._cursor.advance();
}
};
_Tokenizer.prototype._readChar = function (decodeEntities) {
if (decodeEntities && this._cursor.peek() === chars.$AMPERSAND) {
return this._decodeEntity();
}
else {
// Don't rely upon reading directly from `_input` as the actual char value
// may have been generated from an escape sequence.
var char = String.fromCodePoint(this._cursor.peek());
this._cursor.advance();
return char;
}
};
_Tokenizer.prototype._decodeEntity = function () {
var start = this._cursor.clone();
this._cursor.advance();
if (this._attemptCharCode(chars.$HASH)) {
var isHex = this._attemptCharCode(chars.$x) || this._attemptCharCode(chars.$X);
var codeStart = this._cursor.clone();
this._attemptCharCodeUntilFn(isDigitEntityEnd);
if (this._cursor.peek() != chars.$SEMICOLON) {
// Advance cursor to include the peeked character in the string provided to the error
// message.
this._cursor.advance();
var entityType = isHex ? CharacterReferenceType.HEX : CharacterReferenceType.DEC;
throw this._createError(_unparsableEntityErrorMsg(entityType, this._cursor.getChars(start)), this._cursor.getSpan());
}
var strNum = this._cursor.getChars(codeStart);
this._cursor.advance();
try {
var charCode = parseInt(strNum, isHex ? 16 : 10);
return String.fromCharCode(charCode);
}
catch (_a) {
throw this._createError(_unknownEntityErrorMsg(this._cursor.getChars(start)), this._cursor.getSpan());
}
}
else {
var nameStart = this._cursor.clone();
this._attemptCharCodeUntilFn(isNamedEntityEnd);
if (this._cursor.peek() != chars.$SEMICOLON) {
this._cursor = nameStart;
return '&';
}
var name_1 = this._cursor.getChars(nameStart);
this._cursor.advance();
var char = tags_1.NAMED_ENTITIES[name_1];
if (!char) {
throw this._createError(_unknownEntityErrorMsg(name_1), this._cursor.getSpan(start));
}
return char;
}
};
_Tokenizer.prototype._consumeRawText = function (decodeEntities, endMarkerPredicate) {
this._beginToken(decodeEntities ? TokenType.ESCAPABLE_RAW_TEXT : TokenType.RAW_TEXT);
var parts = [];
while (true) {
var tagCloseStart = this._cursor.clone();
var foundEndMarker = endMarkerPredicate();
this._cursor = tagCloseStart;
if (foundEndMarker) {
break;
}
parts.push(this._readChar(decodeEntities));
}
return this._endToken([this._processCarriageReturns(parts.join(''))]);
};
_Tokenizer.prototype._consumeComment = function (start) {
var _this = this;
this._beginToken(TokenType.COMMENT_START, start);
this._requireCharCode(chars.$MINUS);
this._endToken([]);
this._consumeRawText(false, function () { return _this._attemptStr('-->'); });
this._beginToken(TokenType.COMMENT_END);
this._requireStr('-->');
this._endToken([]);
};
_Tokenizer.prototype._consumeCdata = function (start) {
var _this = this;
this._beginToken(TokenType.CDATA_START, start);
this._requireStr('CDATA[');
this._endToken([]);
this._consumeRawText(false, function () { return _this._attemptStr(']]>'); });
this._beginToken(TokenType.CDATA_END);
this._requireStr(']]>');
this._endToken([]);
};
_Tokenizer.prototype._consumeDocType = function (start) {
this._beginToken(TokenType.DOC_TYPE, start);
var contentStart = this._cursor.clone();
this._attemptUntilChar(chars.$GT);
var content = this._cursor.getChars(contentStart);
this._cursor.advance();
this._endToken([content]);
};
_Tokenizer.prototype._consumePrefixAndName = function () {
var nameOrPrefixStart = this._cursor.clone();
var prefix = '';
while (this._cursor.peek() !== chars.$COLON && !isPrefixEnd(this._cursor.peek())) {
this._cursor.advance();
}
var nameStart;
if (this._cursor.peek() === chars.$COLON) {
prefix = this._cursor.getChars(nameOrPrefixStart);
this._cursor.advance();
nameStart = this._cursor.clone();
}
else {
nameStart = nameOrPrefixStart;
}
this._requireCharCodeUntilFn(isNameEnd, prefix === '' ? 0 : 1);
var name = this._cursor.getChars(nameStart);
return [prefix, name];
};
_Tokenizer.prototype._consumeTagOpen = function (start) {
var tagName;
var prefix;
var openTagToken;
try {
if (!chars.isAsciiLetter(this._cursor.peek())) {
throw this._createError(_unexpectedCharacterErrorMsg(this._cursor.peek()), this._cursor.getSpan(start));
}
openTagToken = this._consumeTagOpenStart(start);
prefix = openTagToken.parts[0];
tagName = openTagToken.parts[1];
this._attemptCharCodeUntilFn(isNotWhitespace);
while (this._cursor.peek() !== chars.$SLASH && this._cursor.peek() !== chars.$GT &&
this._cursor.peek() !== chars.$LT) {
this._consumeAttributeName();
this._attemptCharCodeUntilFn(isNotWhitespace);
if (this._attemptCharCode(chars.$EQ)) {
this._attemptCharCodeUntilFn(isNotWhitespace);
this._consumeAttributeValue();
}
this._attemptCharCodeUntilFn(isNotWhitespace);
}
this._consumeTagOpenEnd();
}
catch (e) {
if (e instanceof _ControlFlowError) {
if (openTagToken) {
// We errored before we could close the opening tag, so it is incomplete.
openTagToken.type = TokenType.INCOMPLETE_TAG_OPEN;
}
else {
// When the start tag is invalid, assume we want a "<" as text.
// Back to back text tokens are merged at the end.
this._beginToken(TokenType.TEXT, start);
this._endToken(['<']);
}
return;
}
throw e;
}
var contentTokenType = this._getTagDefinition(tagName).contentType;
if (contentTokenType === tags_1.TagContentType.RAW_TEXT) {
this._consumeRawTextWithTagClose(prefix, tagName, false);
}
else if (contentTokenType === tags_1.TagContentType.ESCAPABLE_RAW_TEXT) {
this._consumeRawTextWithTagClose(prefix, tagName, true);
}
};
_Tokenizer.prototype._consumeRawTextWithTagClose = function (prefix, tagName, decodeEntities) {
var _this = this;
var textToken = this._consumeRawText(decodeEntities, function () {
if (!_this._attemptCharCode(chars.$LT))
return false;
if (!_this._attemptCharCode(chars.$SLASH))
return false;
_this._attemptCharCodeUntilFn(isNotWhitespace);
if (!_this._attemptStrCaseInsensitive(tagName))
return false;
_this._attemptCharCodeUntilFn(isNotWhitespace);
return _this._attemptCharCode(chars.$GT);
});
this._beginToken(TokenType.TAG_CLOSE);
this._requireCharCodeUntilFn(function (code) { return code === chars.$GT; }, 3);
this._cursor.advance(); // Consume the `>`
this._endToken([prefix, tagName]);
};
_Tokenizer.prototype._consumeTagOpenStart = function (start) {
this._beginToken(TokenType.TAG_OPEN_START, start);
var parts = this._consumePrefixAndName();
return this._endToken(parts);
};
_Tokenizer.prototype._consumeAttributeName = function () {
var attrNameStart = this._cursor.peek();
if (attrNameStart === chars.$SQ || attrNameStart === chars.$DQ) {
throw this._createError(_unexpectedCharacterErrorMsg(attrNameStart), this._cursor.getSpan());
}
this._beginToken(TokenType.ATTR_NAME);
var prefixAndName = this._consumePrefixAndName();
this._endToken(prefixAndName);
};
_Tokenizer.prototype._consumeAttributeValue = function () {
var value;
if (this._cursor.peek() === chars.$SQ || this._cursor.peek() === chars.$DQ) {
this._beginToken(TokenType.ATTR_QUOTE);
var quoteChar = this._cursor.peek();
this._cursor.advance();
this._endToken([String.fromCodePoint(quoteChar)]);
this._beginToken(TokenType.ATTR_VALUE);
var parts = [];
while (this._cursor.peek() !== quoteChar) {
parts.push(this._readChar(true));
}
value = parts.join('');
this._endToken([this._processCarriageReturns(value)]);
this._beginToken(TokenType.ATTR_QUOTE);
this._cursor.advance();
this._endToken([String.fromCodePoint(quoteChar)]);
}
else {
this._beginToken(TokenType.ATTR_VALUE);
var valueStart = this._cursor.clone();
this._requireCharCodeUntilFn(isNameEnd, 1);
value = this._cursor.getChars(valueStart);
this._endToken([this._processCarriageReturns(value)]);
}
};
_Tokenizer.prototype._consumeTagOpenEnd = function () {
var tokenType = this._attemptCharCode(chars.$SLASH) ? TokenType.TAG_OPEN_END_VOID : TokenType.TAG_OPEN_END;
this._beginToken(tokenType);
this._requireCharCode(chars.$GT);
this._endToken([]);
};
_Tokenizer.prototype._consumeTagClose = function (start) {
this._beginToken(TokenType.TAG_CLOSE, start);
this._attemptCharCodeUntilFn(isNotWhitespace);
var prefixAndName = this._consumePrefixAndName();
this._attemptCharCodeUntilFn(isNotWhitespace);
this._requireCharCode(chars.$GT);
this._endToken(prefixAndName);
};
_Tokenizer.prototype._consumeExpansionFormStart = function () {
this._beginToken(TokenType.EXPANSION_FORM_START);
this._requireCharCode(chars.$LBRACE);
this._endToken([]);
this._expansionCaseStack.push(TokenType.EXPANSION_FORM_START);
this._beginToken(TokenType.RAW_TEXT);
var condition = this._readUntil(chars.$COMMA);
var normalizedCondition = this._processCarriageReturns(condition);
if (this._i18nNormalizeLineEndingsInICUs) {
// We explicitly want to normalize line endings for this text.
this._endToken([normalizedCondition]);
}
else {
// We are not normalizing line endings.
var conditionToken = this._endToken([condition]);
if (normalizedCondition !== condition) {
this.nonNormalizedIcuExpressions.push(conditionToken);
}
}
this._requireCharCode(chars.$COMMA);
this._attemptCharCodeUntilFn(isNotWhitespace);
this._beginToken(TokenType.RAW_TEXT);
var type = this._readUntil(chars.$COMMA);
this._endToken([type]);
this._requireCharCode(chars.$COMMA);
this._attemptCharCodeUntilFn(isNotWhitespace);
};
_Tokenizer.prototype._consumeExpansionCaseStart = function () {
this._beginToken(TokenType.EXPANSION_CASE_VALUE);
var value = this._readUntil(chars.$LBRACE).trim();
this._endToken([value]);
this._attemptCharCodeUntilFn(isNotWhitespace);
this._beginToken(TokenType.EXPANSION_CASE_EXP_START);
this._requireCharCode(chars.$LBRACE);
this._endToken([]);
this._attemptCharCodeUntilFn(isNotWhitespace);
this._expansionCaseStack.push(TokenType.EXPANSION_CASE_EXP_START);
};
_Tokenizer.prototype._consumeExpansionCaseEnd = function () {
this._beginToken(TokenType.EXPANSION_CASE_EXP_END);
this._requireCharCode(chars.$RBRACE);
this._endToken([]);
this._attemptCharCodeUntilFn(isNotWhitespace);
this._expansionCaseStack.pop();
};
_Tokenizer.prototype._consumeExpansionFormEnd = function () {
this._beginToken(TokenType.EXPANSION_FORM_END);
this._requireCharCode(chars.$RBRACE);
this._endToken([]);
this._expansionCaseStack.pop();
};
_Tokenizer.prototype._consumeText = function () {
var start = this._cursor.clone();
this._beginToken(TokenType.TEXT, start);
var parts = [];
do {
if (this._interpolationConfig && this._attemptStr(this._interpolationConfig.start)) {
parts.push(this._interpolationConfig.start);
this._inInterpolation = true;
}
else if (this._interpolationConfig && this._inInterpolation &&
this._attemptStr(this._interpolationConfig.end)) {
parts.push(this._interpolationConfig.end);
this._inInterpolation = false;
}
else {
parts.push(this._readChar(true));
}
} while (!this._isTextEnd());
this._endToken([this._processCarriageReturns(parts.join(''))]);
};
_Tokenizer.prototype._isTextEnd = function () {
if (this._cursor.peek() === chars.$LT || this._cursor.peek() === chars.$EOF) {
return true;
}
if (this._tokenizeIcu && !this._inInterpolation) {
if (this.isExpansionFormStart()) {
// start of an expansion form
return true;
}
if (this._cursor.peek() === chars.$RBRACE && this._isInExpansionCase()) {
// end of and expansion case
return true;
}
}
return false;
};
_Tokenizer.prototype._readUntil = function (char) {
var start = this._cursor.clone();
this._attemptUntilChar(char);
return this._cursor.getChars(start);
};
_Tokenizer.prototype._isInExpansionCase = function () {
return this._expansionCaseStack.length > 0 &&
this._expansionCaseStack[this._expansionCaseStack.length - 1] ===
TokenType.EXPANSION_CASE_EXP_START;
};
_Tokenizer.prototype._isInExpansionForm = function () {
return this._expansionCaseStack.length > 0 &&
this._expansionCaseStack[this._expansionCaseStack.length - 1] ===
TokenType.EXPANSION_FORM_START;
};
_Tokenizer.prototype.isExpansionFormStart = function () {
if (this._cursor.peek() !== chars.$LBRACE) {
return false;
}
if (this._interpolationConfig) {
var start = this._cursor.clone();
var isInterpolation = this._attemptStr(this._interpolationConfig.start);
this._cursor = start;
return !isInterpolation;
}
return true;
};
return _Tokenizer;
}());
function isNotWhitespace(code) {
return !chars.isWhitespace(code) || code === chars.$EOF;
}
function isNameEnd(code) {
return chars.isWhitespace(code) || code === chars.$GT || code === chars.$LT ||
code === chars.$SLASH || code === chars.$SQ || code === chars.$DQ || code === chars.$EQ;
}
function isPrefixEnd(code) {
return (code < chars.$a || chars.$z < code) && (code < chars.$A || chars.$Z < code) &&
(code < chars.$0 || code > chars.$9);
}
function isDigitEntityEnd(code) {
return code == chars.$SEMICOLON || code == chars.$EOF || !chars.isAsciiHexDigit(code);
}
function isNamedEntityEnd(code) {
return code == chars.$SEMICOLON || code == chars.$EOF || !chars.isAsciiLetter(code);
}
function isExpansionCaseStart(peek) {
return peek !== chars.$RBRACE;
}
function compareCharCodeCaseInsensitive(code1, code2) {
return toUpperCaseCharCode(code1) == toUpperCaseCharCode(code2);
}
function toUpperCaseCharCode(code) {
return code >= chars.$a && code <= chars.$z ? code - chars.$a + chars.$A : code;
}
function mergeTextTokens(srcTokens) {
var dstTokens = [];
var lastDstToken = undefined;
for (var i = 0; i < srcTokens.length; i++) {
var token = srcTokens[i];
if (lastDstToken && lastDstToken.type == TokenType.TEXT && token.type == TokenType.TEXT) {
lastDstToken.parts[0] += token.parts[0];
lastDstToken.sourceSpan.end = token.sourceSpan.end;
}
else {
lastDstToken = token;
dstTokens.push(lastDstToken);
}
}
return dstTokens;
}
var PlainCharacterCursor = /** @class */ (function () {
function PlainCharacterCursor(fileOrCursor, range) {
if (fileOrCursor instanceof PlainCharacterCursor) {
this.file = fileOrCursor.file;
this.input = fileOrCursor.input;
this.end = fileOrCursor.end;
var state = fileOrCursor.state;
// Note: avoid using `{...fileOrCursor.state}` here as that has a severe performance penalty.
// In ES5 bundles the object spread operator is translated into the `__assign` helper, which
// is not optimized by VMs as efficiently as a raw object literal. Since this constructor is
// called in tight loops, this difference matters.
this.state = {
peek: state.peek,
offset: state.offset,
line: state.line,
column: state.column,
};
}
else {
if (!range) {
throw new Error('Programming error: the range argument must be provided with a file argument.');
}
this.file = fileOrCursor;
this.input = fileOrCursor.content;
this.end = range.endPos;
this.state = {
peek: -1,
offset: range.startPos,
line: range.startLine,
column: range.startCol,
};
}
}
PlainCharacterCursor.prototype.clone = function () {
return new PlainCharacterCursor(this);
};
PlainCharacterCursor.prototype.peek = function () {
return this.state.peek;
};
PlainCharacterCursor.prototype.charsLeft = function () {
return this.end - this.state.offset;
};
PlainCharacterCursor.prototype.diff = function (other) {
return this.state.offset - other.state.offset;
};
PlainCharacterCursor.prototype.advance = function () {
this.advanceState(this.state);
};
PlainCharacterCursor.prototype.init = function () {
this.updatePeek(this.state);
};
PlainCharacterCursor.prototype.getSpan = function (start, leadingTriviaCodePoints) {
start = start || this;
var fullStart = start;
if (leadingTriviaCodePoints) {
while (this.diff(start) > 0 && leadingTriviaCodePoints.indexOf(start.peek()) !== -1) {
if (fullStart === start) {
start = start.clone();
}
start.advance();
}
}
var startLocation = this.locationFromCursor(start);
var endLocation = this.locationFromCursor(this);
var fullStartLocation = fullStart !== start ? this.locationFromCursor(fullStart) : startLocation;
return new parse_util_1.ParseSourceSpan(startLocation, endLocation, fullStartLocation);
};
PlainCharacterCursor.prototype.getChars = function (start) {
return this.input.substring(start.state.offset, this.state.offset);
};
PlainCharacterCursor.prototype.charAt = function (pos) {
return this.input.charCodeAt(pos);
};
PlainCharacterCursor.prototype.advanceState = function (state) {
if (state.offset >= this.end) {
this.state = state;
throw new CursorError('Unexpected character "EOF"', this);
}
var currentChar = this.charAt(state.offset);
if (currentChar === chars.$LF) {
state.line++;
state.column = 0;
}
else if (!chars.isNewLine(currentChar)) {
state.column++;
}
state.offset++;
this.updatePeek(state);
};
PlainCharacterCursor.prototype.updatePeek = function (state) {
state.peek = state.offset >= this.end ? chars.$EOF : this.charAt(state.offset);
};
PlainCharacterCursor.prototype.locationFromCursor = function (cursor) {
return new parse_util_1.ParseLocation(cursor.file, cursor.state.offset, cursor.state.line, cursor.state.column);
};
return PlainCharacterCursor;
}());
var EscapedCharacterCursor = /** @class */ (function (_super) {
tslib_1.__extends(EscapedCharacterCursor, _super);
function EscapedCharacterCursor(fileOrCursor, range) {
var _this = this;
if (fileOrCursor instanceof EscapedCharacterCursor) {
_this = _super.call(this, fileOrCursor) || this;
_this.internalState = tslib_1.__assign({}, fileOrCursor.internalState);
}
else {
_this = _super.call(this, fileOrCursor, range) || this;
_this.internalState = _this.state;
}
return _this;
}
EscapedCharacterCursor.prototype.advance = function () {
this.state = this.internalState;
_super.prototype.advance.call(this);
this.processEscapeSequence();
};
EscapedCharacterCursor.prototype.init = function () {
_super.prototype.init.call(this);
this.processEscapeSequence();
};
EscapedCharacterCursor.prototype.clone = function () {
return new EscapedCharacterCursor(this);
};
EscapedCharacterCursor.prototype.getChars = function (start) {
var cursor = start.clone();
var chars = '';
while (cursor.internalState.offset < this.internalState.offset) {
chars += String.fromCodePoint(cursor.peek());
cursor.advance();
}
return chars;
};
/**
* Process the escape sequence that starts at the current position in the text.
*
* This method is called to ensure that `peek` has the unescaped value of escape sequences.
*/
EscapedCharacterCursor.prototype.processEscapeSequence = function () {
var _this = this;
var peek = function () { return _this.internalState.peek; };
if (peek() === chars.$BACKSLASH) {
// We have hit an escape sequence so we need the internal state to become independent
// of the external state.
this.internalState = tslib_1.__assign({}, this.state);
// Move past the backslash
this.advanceState(this.internalState);
// First check for standard control char sequences
if (peek() === chars.$n) {
this.state.peek = chars.$LF;
}
else if (peek() === chars.$r) {
this.state.peek = chars.$CR;
}
else if (peek() === chars.$v) {
this.state.peek = chars.$VTAB;
}
else if (peek() === chars.$t) {
this.state.peek = chars.$TAB;
}
else if (peek() === chars.$b) {
this.state.peek = chars.$BSPACE;
}
else if (peek() === chars.$f) {
this.state.peek = chars.$FF;
}
// Now consider more complex sequences
else if (peek() === chars.$u) {
// Unicode code-point sequence
this.advanceState(this.internalState); // advance past the `u` char
if (peek() === chars.$LBRACE) {
// Variable length Unicode, e.g. `\x{123}`
this.advanceState(this.internalState); // advance past the `{` char
// Advance past the variable number of hex digits until we hit a `}` char
var digitStart = this.clone();
var length_1 = 0;
while (peek() !== chars.$RBRACE) {
this.advanceState(this.internalState);
length_1++;
}
this.state.peek = this.decodeHexDigits(digitStart, length_1);
}
else {
// Fixed length Unicode, e.g. `\u1234`
var digitStart = this.clone();
this.advanceState(this.internalState);
this.advanceState(this.internalState);
this.advanceState(this.internalState);
this.state.peek = this.decodeHexDigits(digitStart, 4);
}
}
else if (peek() === chars.$x) {
// Hex char code, e.g. `\x2F`
this.advanceState(this.internalState); // advance past the `x` char
var digitStart = this.clone();
this.advanceState(this.internalState);
this.state.peek = this.decodeHexDigits(digitStart, 2);
}
else if (chars.isOctalDigit(peek())) {
// Octal char code, e.g. `\012`,
var octal = '';
var length_2 = 0;
var previous = this.clone();
while (chars.isOctalDigit(peek()) && length_2 < 3) {
previous = this.clone();
octal += String.fromCodePoint(peek());
this.advanceState(this.internalState);
length_2++;
}
this.state.peek = parseInt(octal, 8);
// Backup one char
this.internalState = previous.internalState;
}
else if (chars.isNewLine(this.internalState.peek)) {
// Line continuation `\` followed by a new line
this.advanceState(this.internalState); // advance over the newline
this.state = this.internalState;
}
else {
// If none of the `if` blocks were executed then we just have an escaped normal character.
// In that case we just, effectively, skip the backslash from the character.
this.state.peek = this.internalState.peek;
}
}
};
EscapedCharacterCursor.prototype.decodeHexDigits = function (start, length) {
var hex = this.input.substr(start.internalState.offset, length);
var charCode = parseInt(hex, 16);
if (!isNaN(charCode)) {
return charCode;
}
else {
start.state = start.internalState;
throw new CursorError('Invalid hexadecimal escape sequence', start);
}
};
return EscapedCharacterCursor;
}(PlainCharacterCursor));
var CursorError = /** @class */ (function () {
function CursorError(msg, cursor) {
this.msg = msg;
this.cursor = cursor;
}
return CursorError;
}());
exports.CursorError = CursorError;
});
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGV4ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi8uLi9wYWNrYWdlcy9jb21waWxlci9zcmMvbWxfcGFyc2VyL2xleGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOzs7Ozs7R0FNRzs7Ozs7Ozs7Ozs7Ozs7SUFFSCxtREFBa0M7SUFDbEMsK0RBQTBGO0lBRTFGLDZGQUF5RjtJQUN6Riw2REFBcUU7SUFFckUsSUFBWSxTQXVCWDtJQXZCRCxXQUFZLFNBQVM7UUFDbkIsNkRBQWMsQ0FBQTtRQUNkLHlEQUFZLENBQUE7UUFDWixtRUFBaUIsQ0FBQTtRQUNqQixtREFBUyxDQUFBO1FBQ1QsdUVBQW1CLENBQUE7UUFDbkIseUNBQUksQ0FBQTtRQUNKLHFFQUFrQixDQUFBO1FBQ2xCLGlEQUFRLENBQUE7UUFDUiwyREFBYSxDQUFBO1FBQ2IsdURBQVcsQ0FBQTtRQUNYLHdEQUFXLENBQUE7UUFDWCxvREFBUyxDQUFBO1FBQ1Qsb0RBQVMsQ0FBQTtRQUNULHNEQUFVLENBQUE7UUFDVixzREFBVSxDQUFBO1FBQ1Ysa0RBQVEsQ0FBQTtRQUNSLDBFQUFvQixDQUFBO1FBQ3BCLDBFQUFvQixDQUFBO1FBQ3BCLGtGQUF3QixDQUFBO1FBQ3hCLDhFQUFzQixDQUFBO1FBQ3RCLHNFQUFrQixDQUFBO1FBQ2xCLHdDQUFHLENBQUE7SUFDTCxDQUFDLEVBdkJXLFNBQVMsR0FBVCxpQkFBUyxLQUFULGlCQUFTLFFBdUJwQjtJQUVEO1FBQ0UsZUFDVyxJQUFvQixFQUFTLEtBQWUsRUFBUyxVQUEyQjtZQUFoRixTQUFJLEdBQUosSUFBSSxDQUFnQjtZQUFTLFVBQUssR0FBTCxLQUFLLENBQVU7WUFBUyxlQUFVLEdBQVYsVUFBVSxDQUFpQjtRQUFHLENBQUM7UUFDakcsWUFBQztJQUFELENBQUMsQUFIRCxJQUdDO0lBSFksc0JBQUs7SUFLbEI7UUFBZ0Msc0NBQVU7UUFDeEMsb0JBQVksUUFBZ0IsRUFBUyxTQUF5QixFQUFFLElBQXFCO1lBQXJGLFlBQ0Usa0JBQU0sSUFBSSxFQUFFLFFBQVEsQ0FBQyxTQUN0QjtZQUZvQyxlQUFTLEdBQVQsU0FBUyxDQUFnQjs7UUFFOUQsQ0FBQztRQUNILGlCQUFDO0lBQUQsQ0FBQyxBQUpELENBQWdDLHVCQUFVLEdBSXpDO0lBSlksZ0NBQVU7SUFNdkI7UUFDRSx3QkFDVyxNQUFlLEVBQVMsTUFBb0IsRUFDNUMsMkJBQW9DO1lBRHBDLFdBQU0sR0FBTixNQUFNLENBQVM7WUFBUyxXQUFNLEdBQU4sTUFBTSxDQUFjO1lBQzVDLGdDQUEyQixHQUEzQiwyQkFBMkIsQ0FBUztRQUFHLENBQUM7UUFDckQscUJBQUM7SUFBRCxDQUFDLEFBSkQsSUFJQztJQUpZLHdDQUFjO0lBdUUzQixTQUFnQixRQUFRLENBQ3BCLE1BQWMsRUFBRSxHQUFXLEVBQUUsZ0JBQW9ELEVBQ2pGLE9BQTZCO1FBQTdCLHdCQUFBLEVBQUEsWUFBNkI7UUFDL0IsSUFBTSxTQUFTLEdBQUcsSUFBSSxVQUFVLENBQUMsSUFBSSw0QkFBZSxDQUFDLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxnQkFBZ0IsRUFBRSxPQUFPLENBQUMsQ0FBQztRQUM5RixTQUFTLENBQUMsUUFBUSxFQUFFLENBQUM7UUFDckIsT0FBTyxJQUFJLGNBQWMsQ0FDckIsZUFBZSxDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUMsRUFBRSxTQUFTLENBQUMsTUFBTSxFQUFFLFNBQVMsQ0FBQywyQkFBMkIsQ0FBQyxDQUFDO0lBQ2xHLENBQUM7SUFQRCw0QkFPQztJQUVELElBQU0sa0JBQWtCLEdBQUcsUUFBUSxDQUFDO0lBRXBDLFNBQVMsNEJBQTRCLENBQUMsUUFBZ0I7UUFDcEQsSUFBTSxJQUFJLEdBQUcsUUFBUSxLQUFLLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLFlBQVksQ0FBQyxRQUFRLENBQUMsQ0FBQztRQUM3RSxPQUFPLDRCQUF5QixJQUFJLE9BQUcsQ0FBQztJQUMxQyxDQUFDO0lBRUQsU0FBUyxzQkFBc0IsQ0FBQyxTQUFpQjtRQUMvQyxPQUFPLHNCQUFtQixTQUFTLDJEQUFtRCxDQUFDO0lBQ3pGLENBQUM7SUFFRCxTQUFTLHlCQUF5QixDQUFDLElBQTRCLEVBQUUsU0FBaUI7UUFDaEYsT0FBTyw4QkFBMkIsU0FBUyxhQUN2QyxJQUFJLHNEQUFpRCxDQUFDO0lBQzVELENBQUM7SUFFRCxJQUFLLHNCQUdKO0lBSEQsV0FBSyxzQkFBc0I7UUFDekIsNkNBQW1CLENBQUE7UUFDbkIseUNBQWUsQ0FBQTtJQUNqQixDQUFDLEVBSEksc0JBQXNCLEtBQXRCLHNCQUFzQixRQUcxQjtJQUVEO1FBQ0UsMkJBQW1CLEtBQWlCO1lBQWpCLFVBQUssR0FBTCxLQUFLLENBQVk7UUFBRyxDQUFDO1FBQzFDLHdCQUFDO0lBQUQsQ0FBQyxBQUZELElBRUM7SUFFRCxzREFBc0Q7SUFDdEQ7UUFnQkU7Ozs7V0FJRztRQUNILG9CQUNJLEtBQXNCLEVBQVUsaUJBQXFELEVBQ3JGLE9BQXdCO1lBRFEsc0JBQWlCLEdBQWpCLGlCQUFpQixDQUFvQztZQWpCakYsdUJBQWtCLEdBQXlCLElBQUksQ0FBQztZQUNoRCxzQkFBaUIsR0FBbUIsSUFBSSxDQUFDO1lBQ3pDLHdCQUFtQixHQUFnQixFQUFFLENBQUM7WUFDdEMscUJBQWdCLEdBQVksS0FBSyxDQUFDO1lBSTFDLFdBQU0sR0FBWSxFQUFFLENBQUM7WUFDckIsV0FBTSxHQUFpQixFQUFFLENBQUM7WUFDMUIsZ0NBQTJCLEdBQVksRUFBRSxDQUFDO1lBVXhDLElBQUksQ0FBQyxZQUFZLEdBQUcsT0FBTyxDQUFDLHNCQUFzQixJQUFJLEtBQUssQ0FBQztZQUM1RCxJQUFJLENBQUMsb0JBQW9CLEdBQUcsT0FBTyxDQUFDLG1CQUFtQixJQUFJLG1EQUE0QixDQUFDO1lBQ3hGLElBQUksQ0FBQyx3QkFBd0I7Z0JBQ3pCLE9BQU8sQ0FBQyxrQkFBa0IsSUFBSSxPQUFPLENBQUMsa0JBQWtCLENBQUMsR0FBRyxDQUFDLFVBQUEsQ0FBQyxJQUFJLE9BQUEsQ0FBQyxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLEVBQXJCLENBQXFCLENBQUMsQ0FBQztZQUM3RixJQUFNLEtBQUssR0FDUCxPQUFPLENBQUMsS0FBSyxJQUFJLEVBQUMsTUFBTSxFQUFFLEtBQUssQ0FBQyxPQUFPLENBQUMsTUFBTSxFQUFFLFFBQVEsRUFBRSxDQUFDLEVBQUUsU0FBUyxFQUFFLENBQUMsRUFBRSxRQUFRLEVBQUUsQ0FBQyxFQUFDLENBQUM7WUFDNUYsSUFBSSxDQUFDLE9BQU8sR0FBRyxPQUFPLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxJQUFJLHNCQUFzQixDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsQ0FBQyxDQUFDO2dCQUMxQyxJQUFJLG9CQUFvQixDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsQ0FBQztZQUM5RSxJQUFJLENBQUMsb0JBQW9CLEdBQUcsT0FBTyxDQUFDLG1CQUFtQixJQUFJLEtBQUssQ0FBQztZQUNqRSxJQUFJLENBQUMsY0FBYyxHQUFHLE9BQU8sQ0FBQyxhQUFhLElBQUksS0FBSyxDQUFDO1lBQ3JELElBQUksQ0FBQywrQkFBK0IsR0FBRyxPQUFPLENBQUMsOEJBQThCLElBQUksS0FBSyxDQUFDO1lBQ3ZGLElBQUk7Z0JBQ0YsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQzthQUNyQjtZQUFDLE9BQU8sQ0FBQyxFQUFFO2dCQUNWLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUM7YUFDckI7UUFDSCxDQUFDO1FBRU8sNENBQXVCLEdBQS9CLFVBQWdDLE9BQWU7WUFDN0MsSUFBSSxJQUFJLENBQUMsb0JBQW9CLEVBQUU7Z0JBQzdCLE9BQU8sT0FBTyxDQUFDO2FBQ2hCO1lBQ0Qsd0VBQXdFO1lBQ3hFLG1FQUFtRTtZQUNuRSxrQkFBa0I7WUFDbEIsbUVBQW1FO1lBQ25FLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxrQkFBa0IsRUFBRSxJQUFJLENBQUMsQ0FBQztRQUNuRCxDQUFDO1FBRUQsNkJBQVEsR0FBUjtZQUNFLE9BQU8sSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLEVBQUUsS0FBSyxLQUFLLENBQUMsSUFBSSxFQUFFO2dCQUN6QyxJQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDO2dCQUNuQyxJQUFJO29CQUNGLElBQUksSUFBSSxDQUFDLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsRUFBRTt3QkFDcEMsSUFBSSxJQUFJLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxFQUFFOzRCQUN0QyxJQUFJLElBQUksQ0FBQyxnQkFBZ