sucrase
Version:
Super-fast alternative to Babel for when you can target modern JS runtimes
1,051 lines (1,050 loc) • 39.3 kB
JavaScript
"use strict";
/* eslint max-len: 0 */
Object.defineProperty(exports, "__esModule", { value: true });
const base_1 = require("../parser/base");
const charCodes = require("../util/charcodes");
const identifier_1 = require("../util/identifier");
const whitespace_1 = require("../util/whitespace");
const context_1 = require("./context");
const state_1 = require("./state");
const types_1 = require("./types");
// The following character codes are forbidden from being
// an immediate sibling of NumericLiteralSeparator _
const forbiddenNumericSeparatorSiblings = {
decBinOct: [
charCodes.dot,
charCodes.uppercaseB,
charCodes.uppercaseE,
charCodes.uppercaseO,
charCodes.underscore,
charCodes.lowercaseB,
charCodes.lowercaseE,
charCodes.lowercaseO,
],
hex: [
charCodes.dot,
charCodes.uppercaseX,
charCodes.underscore,
charCodes.lowercaseX,
],
};
// tslint:disable-next-line no-any
const allowedNumericSeparatorSiblings = {};
allowedNumericSeparatorSiblings.bin = [
// 0 - 1
charCodes.digit0,
charCodes.digit1,
];
allowedNumericSeparatorSiblings.oct = [
// 0 - 7
...allowedNumericSeparatorSiblings.bin,
charCodes.digit2,
charCodes.digit3,
charCodes.digit4,
charCodes.digit5,
charCodes.digit6,
charCodes.digit7,
];
allowedNumericSeparatorSiblings.dec = [
// 0 - 9
...allowedNumericSeparatorSiblings.oct,
charCodes.digit8,
charCodes.digit9,
];
allowedNumericSeparatorSiblings.hex = [
// 0 - 9, A - F, a - f,
...allowedNumericSeparatorSiblings.dec,
charCodes.uppercaseA,
charCodes.uppercaseB,
charCodes.uppercaseC,
charCodes.uppercaseD,
charCodes.uppercaseE,
charCodes.uppercaseF,
charCodes.lowercaseA,
charCodes.lowercaseB,
charCodes.lowercaseC,
charCodes.lowercaseD,
charCodes.lowercaseE,
charCodes.lowercaseF,
];
var IdentifierRole;
(function (IdentifierRole) {
IdentifierRole[IdentifierRole["Access"] = 0] = "Access";
IdentifierRole[IdentifierRole["ExportAccess"] = 1] = "ExportAccess";
IdentifierRole[IdentifierRole["FunctionScopedDeclaration"] = 2] = "FunctionScopedDeclaration";
IdentifierRole[IdentifierRole["BlockScopedDeclaration"] = 3] = "BlockScopedDeclaration";
IdentifierRole[IdentifierRole["ObjectShorthand"] = 4] = "ObjectShorthand";
IdentifierRole[IdentifierRole["ObjectKey"] = 5] = "ObjectKey";
})(IdentifierRole = exports.IdentifierRole || (exports.IdentifierRole = {}));
// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
class Token {
constructor(state) {
this.type = state.type;
this.value = state.value;
this.start = state.start;
this.end = state.end;
this.isType = state.isType;
}
}
exports.Token = Token;
// ## Tokenizer
function codePointToString(code) {
// UTF-16 Decoding
if (code <= 0xffff) {
return String.fromCharCode(code);
}
else {
return String.fromCharCode(((code - 0x10000) >> 10) + 0xd800, ((code - 0x10000) & 1023) + 0xdc00);
}
}
class Tokenizer extends base_1.default {
constructor(options, input) {
super();
this.state = new state_1.default();
this.state.init(options, input);
this.isLookahead = false;
this.nextContextId = 1;
}
// Move to the next token
next() {
if (this.options.tokens && !this.isLookahead) {
this.state.tokens.push(new Token(this.state));
}
this.state.lastTokEnd = this.state.end;
this.nextToken();
}
runInTypeContext(existingTokensInType, func) {
for (let i = this.state.tokens.length - existingTokensInType; i < this.state.tokens.length; i++) {
this.state.tokens[i].isType = true;
}
const oldIsType = this.state.isType;
this.state.isType = true;
const result = func();
this.state.isType = oldIsType;
return result;
}
// TODO
eat(type) {
if (this.match(type)) {
this.next();
return true;
}
else {
return false;
}
}
// TODO
match(type) {
return this.state.type === type;
}
// TODO
isKeyword(word) {
return identifier_1.isKeyword(word);
}
// TODO
lookaheadType() {
const snapshot = this.state.snapshot();
this.isLookahead = true;
this.next();
const type = this.state.type;
this.isLookahead = false;
this.state.restoreFromSnapshot(snapshot);
return type;
}
// tslint:disable-next-line no-any
lookaheadTypeAndValue() {
const snapshot = this.state.snapshot();
this.isLookahead = true;
this.next();
const type = this.state.type;
const value = this.state.value;
this.isLookahead = false;
this.state.restoreFromSnapshot(snapshot);
return { type, value };
}
curContext() {
return this.state.context[this.state.context.length - 1];
}
// Read a single token, updating the parser object's token-related
// properties.
nextToken() {
const curContext = this.curContext();
if (!curContext || !curContext.preserveSpace)
this.skipSpace();
this.state.start = this.state.pos;
if (this.state.pos >= this.input.length) {
const tokens = this.state.tokens;
// We normally run past the end a bit, but if we're way past the end, avoid an infinite loop.
// Also check the token positions rather than the types since sometimes we rewrite the token
// type to something else.
if (tokens.length >= 2 &&
tokens[tokens.length - 1].start >= this.input.length &&
tokens[tokens.length - 2].start >= this.input.length) {
this.unexpected(null, "Unexpectedly reached the end of input.");
}
this.finishToken(types_1.types.eof);
return;
}
if (curContext.override) {
curContext.override(this);
}
else {
this.readToken(this.fullCharCodeAtPos());
}
}
readToken(code) {
// Identifier or keyword. '\uXXXX' sequences are allowed in
// identifiers, so '\' also dispatches to that.
if (identifier_1.isIdentifierStart(code) || code === charCodes.backslash) {
this.readWord();
}
else {
this.getTokenFromCode(code);
}
}
fullCharCodeAtPos() {
const code = this.input.charCodeAt(this.state.pos);
if (code <= 0xd7ff || code >= 0xe000)
return code;
const next = this.input.charCodeAt(this.state.pos + 1);
return (code << 10) + next - 0x35fdc00;
}
skipBlockComment() {
const end = this.input.indexOf("*/", (this.state.pos += 2));
if (end === -1)
this.raise(this.state.pos - 2, "Unterminated comment");
this.state.pos = end + 2;
}
skipLineComment(startSkip) {
let ch = this.input.charCodeAt((this.state.pos += startSkip));
if (this.state.pos < this.input.length) {
while (ch !== charCodes.lineFeed &&
ch !== charCodes.carriageReturn &&
ch !== charCodes.lineSeparator &&
ch !== charCodes.paragraphSeparator &&
++this.state.pos < this.input.length) {
ch = this.input.charCodeAt(this.state.pos);
}
}
}
// Called at the start of the parse and after every token. Skips
// whitespace and comments.
skipSpace() {
loop: while (this.state.pos < this.input.length) {
const ch = this.input.charCodeAt(this.state.pos);
switch (ch) {
case charCodes.space:
case charCodes.nonBreakingSpace:
++this.state.pos;
break;
case charCodes.carriageReturn:
if (this.input.charCodeAt(this.state.pos + 1) === charCodes.lineFeed) {
++this.state.pos;
}
case charCodes.lineFeed:
case charCodes.lineSeparator:
case charCodes.paragraphSeparator:
++this.state.pos;
break;
case charCodes.slash:
switch (this.input.charCodeAt(this.state.pos + 1)) {
case charCodes.asterisk:
this.skipBlockComment();
break;
case charCodes.slash:
this.skipLineComment(2);
break;
default:
break loop;
}
break;
default:
if ((ch > charCodes.backSpace && ch < charCodes.shiftOut) ||
(ch >= charCodes.oghamSpaceMark && whitespace_1.nonASCIIwhitespace.test(String.fromCharCode(ch)))) {
++this.state.pos;
}
else {
break loop;
}
}
}
}
// Called at the end of every token. Sets `end`, `val`, and
// maintains `context` and `exprAllowed`, and skips the space after
// the token, so that the next one's `start` will point at the
// right position.
// tslint:disable-next-line no-any
finishToken(type, val) {
this.state.end = this.state.pos;
const prevType = this.state.type;
this.state.type = type;
this.state.value = val;
this.updateContext(prevType);
}
// ### Token reading
// This is the function that is called to fetch the next token. It
// is somewhat obscure, because it works in character codes rather
// than characters, and because operator parsing has been inlined
// into it.
//
// All in the name of speed.
//
readToken_dot() {
const next = this.input.charCodeAt(this.state.pos + 1);
if (next >= charCodes.digit0 && next <= charCodes.digit9) {
this.readNumber(true);
return;
}
const next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === charCodes.dot && next2 === charCodes.dot) {
this.state.pos += 3;
this.finishToken(types_1.types.ellipsis);
}
else {
++this.state.pos;
this.finishToken(types_1.types.dot);
}
}
readToken_slash() {
// '/'
if (this.state.exprAllowed) {
++this.state.pos;
this.readRegexp();
return;
}
const next = this.input.charCodeAt(this.state.pos + 1);
if (next === charCodes.equalsTo) {
this.finishOp(types_1.types.assign, 2);
}
else {
this.finishOp(types_1.types.slash, 1);
}
}
readToken_mult_modulo(code) {
// '%*'
let type = code === charCodes.asterisk ? types_1.types.star : types_1.types.modulo;
let width = 1;
let next = this.input.charCodeAt(this.state.pos + 1);
const exprAllowed = this.state.exprAllowed;
// Exponentiation operator **
if (code === charCodes.asterisk && next === charCodes.asterisk) {
width++;
next = this.input.charCodeAt(this.state.pos + 2);
type = types_1.types.exponent;
}
if (next === charCodes.equalsTo && !exprAllowed) {
width++;
type = types_1.types.assign;
}
this.finishOp(type, width);
}
readToken_pipe_amp(code) {
// '|&'
const next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) {
this.finishOp(code === charCodes.verticalBar ? types_1.types.logicalOR : types_1.types.logicalAND, 2);
return;
}
if (code === charCodes.verticalBar) {
// '|>'
if (next === charCodes.greaterThan) {
this.finishOp(types_1.types.pipeline, 2);
return;
}
else if (next === charCodes.rightCurlyBrace && this.hasPlugin("flow")) {
// '|}'
this.finishOp(types_1.types.braceBarR, 2);
return;
}
}
if (next === charCodes.equalsTo) {
this.finishOp(types_1.types.assign, 2);
return;
}
this.finishOp(code === charCodes.verticalBar ? types_1.types.bitwiseOR : types_1.types.bitwiseAND, 1);
}
readToken_caret() {
// '^'
const next = this.input.charCodeAt(this.state.pos + 1);
if (next === charCodes.equalsTo) {
this.finishOp(types_1.types.assign, 2);
}
else {
this.finishOp(types_1.types.bitwiseXOR, 1);
}
}
readToken_plus_min(code) {
// '+-'
const next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) {
this.finishOp(types_1.types.incDec, 2);
return;
}
if (next === charCodes.equalsTo) {
this.finishOp(types_1.types.assign, 2);
}
else {
this.finishOp(types_1.types.plusMin, 1);
}
}
readToken_lt_gt(code) {
// '<>'
const next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) {
const size = code === charCodes.greaterThan &&
this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan
? 3
: 2;
if (this.input.charCodeAt(this.state.pos + size) === charCodes.equalsTo) {
this.finishOp(types_1.types.assign, size + 1);
return;
}
this.finishOp(types_1.types.bitShift, size);
return;
}
if (next === charCodes.equalsTo) {
// <= | >=
this.finishOp(types_1.types.relationalOrEqual, 2);
}
else if (code === charCodes.lessThan) {
this.finishOp(types_1.types.lessThan, 1);
}
else {
this.finishOp(types_1.types.greaterThan, 1);
}
}
readToken_eq_excl(code) {
// '=!'
const next = this.input.charCodeAt(this.state.pos + 1);
if (next === charCodes.equalsTo) {
this.finishOp(types_1.types.equality, this.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo ? 3 : 2);
return;
}
if (code === charCodes.equalsTo && next === charCodes.greaterThan) {
// '=>'
this.state.pos += 2;
this.finishToken(types_1.types.arrow);
return;
}
this.finishOp(code === charCodes.equalsTo ? types_1.types.eq : types_1.types.bang, 1);
}
readToken_question() {
// '?'
const next = this.input.charCodeAt(this.state.pos + 1);
const next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === charCodes.questionMark) {
// '??'
this.finishOp(types_1.types.nullishCoalescing, 2);
}
else if (next === charCodes.dot &&
!(next2 >= charCodes.digit0 && next2 <= charCodes.digit9)) {
// '.' not followed by a number
this.state.pos += 2;
this.finishToken(types_1.types.questionDot);
}
else {
++this.state.pos;
this.finishToken(types_1.types.question);
}
}
getTokenFromCode(code) {
switch (code) {
case charCodes.numberSign:
++this.state.pos;
this.finishToken(types_1.types.hash);
return;
// The interpretation of a dot depends on whether it is followed
// by a digit or another two dots.
case charCodes.dot:
this.readToken_dot();
return;
// Punctuation tokens.
case charCodes.leftParenthesis:
++this.state.pos;
this.finishToken(types_1.types.parenL);
return;
case charCodes.rightParenthesis:
++this.state.pos;
this.finishToken(types_1.types.parenR);
return;
case charCodes.semicolon:
++this.state.pos;
this.finishToken(types_1.types.semi);
return;
case charCodes.comma:
++this.state.pos;
this.finishToken(types_1.types.comma);
return;
case charCodes.leftSquareBracket:
++this.state.pos;
this.finishToken(types_1.types.bracketL);
return;
case charCodes.rightSquareBracket:
++this.state.pos;
this.finishToken(types_1.types.bracketR);
return;
case charCodes.leftCurlyBrace:
if (this.hasPlugin("flow") &&
this.input.charCodeAt(this.state.pos + 1) === charCodes.verticalBar) {
this.finishOp(types_1.types.braceBarL, 2);
}
else {
++this.state.pos;
this.finishToken(types_1.types.braceL);
}
return;
case charCodes.rightCurlyBrace:
++this.state.pos;
this.finishToken(types_1.types.braceR);
return;
case charCodes.colon:
if (this.input.charCodeAt(this.state.pos + 1) === charCodes.colon) {
this.finishOp(types_1.types.doubleColon, 2);
}
else {
++this.state.pos;
this.finishToken(types_1.types.colon);
}
return;
case charCodes.questionMark:
this.readToken_question();
return;
case charCodes.atSign:
++this.state.pos;
this.finishToken(types_1.types.at);
return;
case charCodes.graveAccent:
++this.state.pos;
this.finishToken(types_1.types.backQuote);
return;
case charCodes.digit0: {
const next = this.input.charCodeAt(this.state.pos + 1);
// '0x', '0X' - hex number
if (next === charCodes.lowercaseX || next === charCodes.uppercaseX) {
this.readRadixNumber(16);
return;
}
// '0o', '0O' - octal number
if (next === charCodes.lowercaseO || next === charCodes.uppercaseO) {
this.readRadixNumber(8);
return;
}
// '0b', '0B' - binary number
if (next === charCodes.lowercaseB || next === charCodes.uppercaseB) {
this.readRadixNumber(2);
return;
}
}
// Anything else beginning with a digit is an integer, octal
// number, or float.
case charCodes.digit1:
case charCodes.digit2:
case charCodes.digit3:
case charCodes.digit4:
case charCodes.digit5:
case charCodes.digit6:
case charCodes.digit7:
case charCodes.digit8:
case charCodes.digit9:
this.readNumber(false);
return;
// Quotes produce strings.
case charCodes.quotationMark:
case charCodes.apostrophe:
this.readString(code);
return;
// Operators are parsed inline in tiny state machines. '=' (charCodes.equalsTo) is
// often referred to. `finishOp` simply skips the amount of
// characters it is given as second argument, and returns a token
// of the type given by its first argument.
case charCodes.slash:
this.readToken_slash();
return;
case charCodes.percentSign:
case charCodes.asterisk:
this.readToken_mult_modulo(code);
return;
case charCodes.verticalBar:
case charCodes.ampersand:
this.readToken_pipe_amp(code);
return;
case charCodes.caret:
this.readToken_caret();
return;
case charCodes.plusSign:
case charCodes.dash:
this.readToken_plus_min(code);
return;
case charCodes.lessThan:
case charCodes.greaterThan:
this.readToken_lt_gt(code);
return;
case charCodes.equalsTo:
case charCodes.exclamationMark:
this.readToken_eq_excl(code);
return;
case charCodes.tilde:
this.finishOp(types_1.types.tilde, 1);
return;
default:
break;
}
this.raise(this.state.pos, `Unexpected character '${codePointToString(code)}'`);
}
finishOp(type, size) {
const str = this.input.slice(this.state.pos, this.state.pos + size);
this.state.pos += size;
this.finishToken(type, str);
}
readRegexp() {
const start = this.state.pos;
let escaped;
let inClass;
for (;;) {
if (this.state.pos >= this.input.length) {
this.raise(start, "Unterminated regular expression");
}
const ch = this.input.charAt(this.state.pos);
if (whitespace_1.lineBreak.test(ch)) {
this.raise(start, "Unterminated regular expression");
}
if (escaped) {
escaped = false;
}
else {
if (ch === "[") {
inClass = true;
}
else if (ch === "]" && inClass) {
inClass = false;
}
else if (ch === "/" && !inClass) {
break;
}
escaped = ch === "\\";
}
++this.state.pos;
}
const content = this.input.slice(start, this.state.pos);
++this.state.pos;
// Need to use `readWord1` because '\uXXXX' sequences are allowed
// here (don't ask).
const mods = this.readWord1();
if (mods) {
const validFlags = /^[gmsiyu]*$/;
if (!validFlags.test(mods)) {
this.raise(start, "Invalid regular expression flag");
}
}
this.finishToken(types_1.types.regexp, {
pattern: content,
flags: mods,
});
}
// Read an integer in the given radix. Return null if zero digits
// were read, the integer value otherwise. When `len` is given, this
// will return `null` unless the integer has exactly `len` digits.
readInt(radix, len) {
const start = this.state.pos;
const forbiddenSiblings = radix === 16
? forbiddenNumericSeparatorSiblings.hex
: forbiddenNumericSeparatorSiblings.decBinOct;
/* eslint-disable no-nested-ternary */
const allowedSiblings = radix === 16
? allowedNumericSeparatorSiblings.hex
: radix === 10
? allowedNumericSeparatorSiblings.dec
: radix === 8 ? allowedNumericSeparatorSiblings.oct : allowedNumericSeparatorSiblings.bin;
/* eslint-enable no-nested-ternary */
let total = 0;
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
const code = this.input.charCodeAt(this.state.pos);
let val;
// Handle numeric separators.
const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1);
if (code === charCodes.underscore) {
if (allowedSiblings.indexOf(next) === -1) {
this.raise(this.state.pos, "Invalid or unexpected token");
}
if (forbiddenSiblings.indexOf(prev) > -1 ||
forbiddenSiblings.indexOf(next) > -1 ||
Number.isNaN(next)) {
this.raise(this.state.pos, "Invalid or unexpected token");
}
// Ignore this _ character
++this.state.pos;
continue;
}
if (code >= charCodes.lowercaseA) {
val = code - charCodes.lowercaseA + charCodes.lineFeed;
}
else if (code >= charCodes.uppercaseA) {
val = code - charCodes.uppercaseA + charCodes.lineFeed;
}
else if (charCodes.isDigit(code)) {
val = code - charCodes.digit0; // 0-9
}
else {
val = Infinity;
}
if (val >= radix)
break;
++this.state.pos;
total = total * radix + val;
}
if (this.state.pos === start || (len != null && this.state.pos - start !== len)) {
return null;
}
return total;
}
readRadixNumber(radix) {
const start = this.state.pos;
let isBigInt = false;
this.state.pos += 2; // 0x
const val = this.readInt(radix);
if (val == null) {
this.raise(this.state.start + 2, `Expected number in radix ${radix}`);
}
if (this.input.charCodeAt(this.state.pos) === charCodes.lowercaseN) {
++this.state.pos;
isBigInt = true;
}
if (identifier_1.isIdentifierStart(this.fullCharCodeAtPos())) {
this.raise(this.state.pos, "Identifier directly after number");
}
if (isBigInt) {
const str = this.input.slice(start, this.state.pos).replace(/[_n]/g, "");
this.finishToken(types_1.types.bigint, str);
return;
}
this.finishToken(types_1.types.num, val);
}
// Read an integer, octal integer, or floating-point number.
readNumber(startsWithDot) {
const start = this.state.pos;
let octal = this.input.charCodeAt(start) === charCodes.digit0;
let isFloat = false;
let isBigInt = false;
if (!startsWithDot && this.readInt(10) === null) {
this.raise(start, "Invalid number");
}
if (octal && this.state.pos === start + 1)
octal = false; // number === 0
let next = this.input.charCodeAt(this.state.pos);
if (next === charCodes.dot && !octal) {
++this.state.pos;
this.readInt(10);
isFloat = true;
next = this.input.charCodeAt(this.state.pos);
}
if ((next === charCodes.uppercaseE || next === charCodes.lowercaseE) && !octal) {
next = this.input.charCodeAt(++this.state.pos);
if (next === charCodes.plusSign || next === charCodes.dash) {
++this.state.pos;
}
if (this.readInt(10) === null)
this.raise(start, "Invalid number");
isFloat = true;
next = this.input.charCodeAt(this.state.pos);
}
if (next === charCodes.lowercaseN) {
// disallow floats and legacy octal syntax, new style octal ("0o") is handled in this.readRadixNumber
if (isFloat || octal)
this.raise(start, "Invalid BigIntLiteral");
++this.state.pos;
isBigInt = true;
}
if (identifier_1.isIdentifierStart(this.fullCharCodeAtPos())) {
this.raise(this.state.pos, "Identifier directly after number");
}
// remove "_" for numeric literal separator, and "n" for BigInts
const str = this.input.slice(start, this.state.pos).replace(/[_n]/g, "");
if (isBigInt) {
this.finishToken(types_1.types.bigint, str);
return;
}
let val;
if (isFloat) {
val = parseFloat(str);
}
else if (!octal || str.length === 1) {
val = parseInt(str, 10);
}
else {
this.raise(start, "Invalid number");
}
this.finishToken(types_1.types.num, val);
}
// Read a string value, interpreting backslash-escapes.
readCodePoint(throwOnInvalid) {
const ch = this.input.charCodeAt(this.state.pos);
let code;
if (ch === charCodes.leftCurlyBrace) {
const codePos = ++this.state.pos;
code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos, throwOnInvalid);
++this.state.pos;
if (code !== null && code > 0x10ffff) {
if (throwOnInvalid) {
this.raise(codePos, "Code point out of bounds");
}
else {
return null;
}
}
}
else {
code = this.readHexChar(4, throwOnInvalid);
}
return code;
}
readString(quote) {
let out = "";
let chunkStart = ++this.state.pos;
for (;;) {
if (this.state.pos >= this.input.length) {
this.raise(this.state.start, "Unterminated string constant");
}
const ch = this.input.charCodeAt(this.state.pos);
if (ch === quote)
break;
if (ch === charCodes.backslash) {
out += this.input.slice(chunkStart, this.state.pos);
// $FlowFixMe
out += this.readEscapedChar(false);
chunkStart = this.state.pos;
}
else {
if (whitespace_1.isNewLine(ch)) {
this.raise(this.state.start, "Unterminated string constant");
}
++this.state.pos;
}
}
out += this.input.slice(chunkStart, this.state.pos++);
this.finishToken(types_1.types.string, out);
}
// Reads template string tokens.
readTmplToken() {
let out = "";
let chunkStart = this.state.pos;
let containsInvalid = false;
for (;;) {
if (this.state.pos >= this.input.length) {
this.raise(this.state.start, "Unterminated template");
}
const ch = this.input.charCodeAt(this.state.pos);
if (ch === charCodes.graveAccent ||
(ch === charCodes.dollarSign &&
this.input.charCodeAt(this.state.pos + 1) === charCodes.leftCurlyBrace)) {
if (this.state.pos === this.state.start && this.match(types_1.types.template)) {
if (ch === charCodes.dollarSign) {
this.state.pos += 2;
this.finishToken(types_1.types.dollarBraceL);
return;
}
else {
++this.state.pos;
this.finishToken(types_1.types.backQuote);
return;
}
}
out += this.input.slice(chunkStart, this.state.pos);
this.finishToken(types_1.types.template, containsInvalid ? null : out);
return;
}
if (ch === charCodes.backslash) {
out += this.input.slice(chunkStart, this.state.pos);
const escaped = this.readEscapedChar(true);
if (escaped === null) {
containsInvalid = true;
}
else {
out += escaped;
}
chunkStart = this.state.pos;
}
else if (whitespace_1.isNewLine(ch)) {
out += this.input.slice(chunkStart, this.state.pos);
++this.state.pos;
switch (ch) {
case charCodes.carriageReturn:
if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) {
++this.state.pos;
}
case charCodes.lineFeed:
out += "\n";
break;
default:
out += String.fromCharCode(ch);
break;
}
chunkStart = this.state.pos;
}
else {
++this.state.pos;
}
}
}
// Used to read escaped characters
readEscapedChar(inTemplate) {
const throwOnInvalid = !inTemplate;
const ch = this.input.charCodeAt(++this.state.pos);
++this.state.pos;
switch (ch) {
case charCodes.lowercaseN:
return "\n";
case charCodes.lowercaseR:
return "\r";
case charCodes.lowercaseX: {
const code = this.readHexChar(2, throwOnInvalid);
return code === null ? null : String.fromCharCode(code);
}
case charCodes.lowercaseU: {
const code = this.readCodePoint(throwOnInvalid);
return code === null ? null : codePointToString(code);
}
case charCodes.lowercaseT:
return "\t";
case charCodes.lowercaseB:
return "\b";
case charCodes.lowercaseV:
return "\u000b";
case charCodes.lowercaseF:
return "\f";
case charCodes.carriageReturn:
if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) {
++this.state.pos;
}
case charCodes.lineFeed:
return "";
default:
// TODO(sucrase): Consider removing all octal parsing.
if (ch >= charCodes.digit0 && ch <= charCodes.digit7) {
const codePos = this.state.pos - 1;
// $FlowFixMe
// @ts-ignore
let octalStr = this.input.substr(this.state.pos - 1, 3).match(/^[0-7]+/)[0];
let octal = parseInt(octalStr, 8);
if (octal > 255) {
octalStr = octalStr.slice(0, -1);
octal = parseInt(octalStr, 8);
}
if (octal > 0) {
if (inTemplate) {
return null;
}
else {
this.raise(codePos, "Octal literal in strict mode");
}
}
this.state.pos += octalStr.length - 1;
return String.fromCharCode(octal);
}
return String.fromCharCode(ch);
}
}
// Used to read character escape sequences ('\x', '\u').
readHexChar(len, throwOnInvalid) {
const codePos = this.state.pos;
const n = this.readInt(16, len);
if (n === null) {
if (throwOnInvalid) {
this.raise(codePos, "Bad character escape sequence");
}
else {
this.state.pos = codePos - 1;
}
}
return n;
}
// Read an identifier, and return it as a string. Sets `this.state.containsEsc`
// to whether the word contained a '\u' escape.
//
// Incrementally adds only escaped chars, adding other chunks as-is
// as a micro-optimization.
readWord1() {
let word = "";
let first = true;
let chunkStart = this.state.pos;
while (this.state.pos < this.input.length) {
const ch = this.fullCharCodeAtPos();
if (identifier_1.isIdentifierChar(ch)) {
this.state.pos += ch <= 0xffff ? 1 : 2;
}
else if (ch === charCodes.backslash) {
word += this.input.slice(chunkStart, this.state.pos);
const escStart = this.state.pos;
if (this.input.charCodeAt(++this.state.pos) !== charCodes.lowercaseU) {
this.raise(this.state.pos, "Expecting Unicode escape sequence \\uXXXX");
}
++this.state.pos;
const esc = this.readCodePoint(true);
// $FlowFixMe (thinks esc may be null, but throwOnInvalid is true)
if (!(first ? identifier_1.isIdentifierStart : identifier_1.isIdentifierChar)(esc)) {
this.raise(escStart, "Invalid Unicode escape");
}
// $FlowFixMe
word += codePointToString(esc);
chunkStart = this.state.pos;
}
else {
break;
}
first = false;
}
return word + this.input.slice(chunkStart, this.state.pos);
}
// Read an identifier or keyword token. Will check for reserved
// words when necessary.
readWord() {
const word = this.readWord1();
let type = types_1.types.name;
if (this.isKeyword(word)) {
type = types_1.keywords[word];
}
this.finishToken(type, word);
}
braceIsBlock(prevType) {
if (prevType === types_1.types.colon) {
const parent = this.curContext();
if (parent === context_1.types.braceStatement || parent === context_1.types.braceExpression) {
return !parent.isExpr;
}
}
if (prevType === types_1.types._return) {
return whitespace_1.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start));
}
if (prevType === types_1.types._else ||
prevType === types_1.types.semi ||
prevType === types_1.types.eof ||
prevType === types_1.types.parenR) {
return true;
}
if (prevType === types_1.types.braceL) {
return this.curContext() === context_1.types.braceStatement;
}
if (prevType === types_1.types.greaterThan) {
// `class C<T> { ... }`
return true;
}
return !this.state.exprAllowed;
}
updateContext(prevType) {
const type = this.state.type;
let update;
if (type.keyword && (prevType === types_1.types.dot || prevType === types_1.types.questionDot)) {
this.state.exprAllowed = false;
// eslint-disable-next-line no-cond-assign
}
else if ((update = type.updateContext)) {
update.call(this, prevType);
}
else {
this.state.exprAllowed = type.beforeExpr;
}
}
}
exports.default = Tokenizer;