coffeescript
Version:
Unfancy JavaScript
1,275 lines (1,240 loc) • 75.9 kB
JavaScript
// Generated by CoffeeScript 2.7.0
(function() {
// The CoffeeScript Lexer. Uses a series of token-matching regexes to attempt
// matches against the beginning of the source code. When a match is found,
// a token is produced, we consume the match, and start again. Tokens are in the
// form:
// [tag, value, locationData]
// where locationData is {first_line, first_column, last_line, last_column, last_line_exclusive, last_column_exclusive}, which is a
// format that can be fed directly into [Jison](https://github.com/zaach/jison). These
// are read by jison in the `parser.lexer` function defined in coffeescript.coffee.
var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARABLE_LEFT_SIDE, COMPARE, COMPOUND_ASSIGN, HERECOMMENT_ILLEGAL, HEREDOC_DOUBLE, HEREDOC_INDENT, HEREDOC_SINGLE, HEREGEX, HEREGEX_COMMENT, HERE_JSTOKEN, IDENTIFIER, INDENTABLE_CLOSERS, INDEXABLE, INSIDE_JSX, INVERSES, JSTOKEN, JSX_ATTRIBUTE, JSX_FRAGMENT_IDENTIFIER, JSX_IDENTIFIER, JSX_IDENTIFIER_PART, JSX_INTERPOLATION, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, Lexer, MATH, MULTI_DENT, NOT_REGEX, NUMBER, OPERATOR, POSSIBLY_DIVISION, REGEX, REGEX_FLAGS, REGEX_ILLEGAL, REGEX_INVALID_ESCAPE, RELATION, RESERVED, Rewriter, SHIFT, STRICT_PROSCRIBED, STRING_DOUBLE, STRING_INVALID_ESCAPE, STRING_SINGLE, STRING_START, TRAILING_SPACES, UNARY, UNARY_MATH, UNFINISHED, VALID_FLAGS, WHITESPACE, addTokenData, attachCommentsToNode, compact, count, flatten, invertLiterate, isForFrom, isUnassignable, key, locationDataToString, merge, parseNumber, repeat, replaceUnicodeCodePointEscapes, starts, throwSyntaxError,
indexOf = [].indexOf,
slice = [].slice;
({Rewriter, INVERSES, UNFINISHED} = require('./rewriter'));
// Import the helpers we need.
({count, starts, compact, repeat, invertLiterate, merge, attachCommentsToNode, locationDataToString, throwSyntaxError, replaceUnicodeCodePointEscapes, flatten, parseNumber} = require('./helpers'));
// The Lexer Class
// ---------------
// The Lexer class reads a stream of CoffeeScript and divvies it up into tagged
// tokens. Some potential ambiguity in the grammar has been avoided by
// pushing some extra smarts into the Lexer.
exports.Lexer = Lexer = class Lexer {
constructor() {
// Throws an error at either a given offset from the current chunk or at the
// location of a token (`token[2]`).
this.error = this.error.bind(this);
}
// **tokenize** is the Lexer's main method. Scan by attempting to match tokens
// one at a time, using a regular expression anchored at the start of the
// remaining code, or a custom recursive token-matching method
// (for interpolations). When the next token has been recorded, we move forward
// within the code past the token, and begin again.
// Each tokenizing method is responsible for returning the number of characters
// it has consumed.
// Before returning the token stream, run it through the [Rewriter](rewriter.html).
tokenize(code, opts = {}) {
var consumed, end, i, ref;
this.literate = opts.literate; // Are we lexing literate CoffeeScript?
this.indent = 0; // The current indentation level.
this.baseIndent = 0; // The overall minimum indentation level.
this.continuationLineAdditionalIndent = 0; // The over-indentation at the current level.
this.outdebt = 0; // The under-outdentation at the current level.
this.indents = []; // The stack of all current indentation levels.
this.indentLiteral = ''; // The indentation.
this.ends = []; // The stack for pairing up tokens.
this.tokens = []; // Stream of parsed tokens in the form `['TYPE', value, location data]`.
this.seenFor = false; // Used to recognize `FORIN`, `FOROF` and `FORFROM` tokens.
this.seenImport = false; // Used to recognize `IMPORT FROM? AS?` tokens.
this.seenExport = false; // Used to recognize `EXPORT FROM? AS?` tokens.
this.importSpecifierList = false; // Used to identify when in an `IMPORT {...} FROM? ...`.
this.exportSpecifierList = false; // Used to identify when in an `EXPORT {...} FROM? ...`.
this.jsxDepth = 0; // Used to optimize JSX checks, how deep in JSX we are.
this.jsxObjAttribute = {}; // Used to detect if JSX attributes is wrapped in {} (<div {props...} />).
this.chunkLine = opts.line || 0; // The start line for the current @chunk.
this.chunkColumn = opts.column || 0; // The start column of the current @chunk.
this.chunkOffset = opts.offset || 0; // The start offset for the current @chunk.
this.locationDataCompensations = opts.locationDataCompensations || {};
code = this.clean(code); // The stripped, cleaned original source code.
// At every position, run through this list of attempted matches,
// short-circuiting if any of them succeed. Their order determines precedence:
// `@literalToken` is the fallback catch-all.
i = 0;
while (this.chunk = code.slice(i)) {
consumed = this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.stringToken() || this.numberToken() || this.jsxToken() || this.regexToken() || this.jsToken() || this.literalToken();
// Update position.
[this.chunkLine, this.chunkColumn, this.chunkOffset] = this.getLineAndColumnFromChunk(consumed);
i += consumed;
if (opts.untilBalanced && this.ends.length === 0) {
return {
tokens: this.tokens,
index: i
};
}
}
this.closeIndentation();
if (end = this.ends.pop()) {
this.error(`missing ${end.tag}`, ((ref = end.origin) != null ? ref : end)[2]);
}
if (opts.rewrite === false) {
return this.tokens;
}
return (new Rewriter()).rewrite(this.tokens);
}
// Preprocess the code to remove leading and trailing whitespace, carriage
// returns, etc. If we’re lexing literate CoffeeScript, strip external Markdown
// by removing all lines that aren’t indented by at least four spaces or a tab.
clean(code) {
var base, thusFar;
thusFar = 0;
if (code.charCodeAt(0) === BOM) {
code = code.slice(1);
this.locationDataCompensations[0] = 1;
thusFar += 1;
}
if (WHITESPACE.test(code)) {
code = `\n${code}`;
this.chunkLine--;
if ((base = this.locationDataCompensations)[0] == null) {
base[0] = 0;
}
this.locationDataCompensations[0] -= 1;
}
code = code.replace(/\r/g, (match, offset) => {
this.locationDataCompensations[thusFar + offset] = 1;
return '';
}).replace(TRAILING_SPACES, '');
if (this.literate) {
code = invertLiterate(code);
}
return code;
}
// Tokenizers
// ----------
// Matches identifying literals: variables, keywords, method names, etc.
// Check to ensure that JavaScript reserved words aren’t being used as
// identifiers. Because CoffeeScript reserves a handful of keywords that are
// allowed in JavaScript, we’re careful not to tag them as keywords when
// referenced as property names here, so you can still do `jQuery.is()` even
// though `is` means `===` otherwise.
identifierToken() {
var alias, colon, colonOffset, colonToken, id, idLength, inJSXTag, input, match, poppedToken, prev, prevprev, ref, ref1, ref10, ref11, ref12, ref2, ref3, ref4, ref5, ref6, ref7, ref8, ref9, regExSuper, regex, sup, tag, tagToken, tokenData;
inJSXTag = this.atJSXTag();
regex = inJSXTag ? JSX_ATTRIBUTE : IDENTIFIER;
if (!(match = regex.exec(this.chunk))) {
return 0;
}
[input, id, colon] = match;
// Preserve length of id for location data
idLength = id.length;
poppedToken = void 0;
if (id === 'own' && this.tag() === 'FOR') {
this.token('OWN', id);
return id.length;
}
if (id === 'from' && this.tag() === 'YIELD') {
this.token('FROM', id);
return id.length;
}
if (id === 'as' && this.seenImport) {
if (this.value() === '*') {
this.tokens[this.tokens.length - 1][0] = 'IMPORT_ALL';
} else if (ref = this.value(true), indexOf.call(COFFEE_KEYWORDS, ref) >= 0) {
prev = this.prev();
[prev[0], prev[1]] = ['IDENTIFIER', this.value(true)];
}
if ((ref1 = this.tag()) === 'DEFAULT' || ref1 === 'IMPORT_ALL' || ref1 === 'IDENTIFIER') {
this.token('AS', id);
return id.length;
}
}
if (id === 'as' && this.seenExport) {
if ((ref2 = this.tag()) === 'IDENTIFIER' || ref2 === 'DEFAULT') {
this.token('AS', id);
return id.length;
}
if (ref3 = this.value(true), indexOf.call(COFFEE_KEYWORDS, ref3) >= 0) {
prev = this.prev();
[prev[0], prev[1]] = ['IDENTIFIER', this.value(true)];
this.token('AS', id);
return id.length;
}
}
if (id === 'default' && this.seenExport && ((ref4 = this.tag()) === 'EXPORT' || ref4 === 'AS')) {
this.token('DEFAULT', id);
return id.length;
}
if (id === 'assert' && (this.seenImport || this.seenExport) && this.tag() === 'STRING') {
this.token('ASSERT', id);
return id.length;
}
if (id === 'do' && (regExSuper = /^(\s*super)(?!\(\))/.exec(this.chunk.slice(3)))) {
this.token('SUPER', 'super');
this.token('CALL_START', '(');
this.token('CALL_END', ')');
[input, sup] = regExSuper;
return sup.length + 3;
}
prev = this.prev();
tag = colon || (prev != null) && (((ref5 = prev[0]) === '.' || ref5 === '?.' || ref5 === '::' || ref5 === '?::') || !prev.spaced && prev[0] === '@') ? 'PROPERTY' : 'IDENTIFIER';
tokenData = {};
if (tag === 'IDENTIFIER' && (indexOf.call(JS_KEYWORDS, id) >= 0 || indexOf.call(COFFEE_KEYWORDS, id) >= 0) && !(this.exportSpecifierList && indexOf.call(COFFEE_KEYWORDS, id) >= 0)) {
tag = id.toUpperCase();
if (tag === 'WHEN' && (ref6 = this.tag(), indexOf.call(LINE_BREAK, ref6) >= 0)) {
tag = 'LEADING_WHEN';
} else if (tag === 'FOR') {
this.seenFor = {
endsLength: this.ends.length
};
} else if (tag === 'UNLESS') {
tag = 'IF';
} else if (tag === 'IMPORT') {
this.seenImport = true;
} else if (tag === 'EXPORT') {
this.seenExport = true;
} else if (indexOf.call(UNARY, tag) >= 0) {
tag = 'UNARY';
} else if (indexOf.call(RELATION, tag) >= 0) {
if (tag !== 'INSTANCEOF' && this.seenFor) {
tag = 'FOR' + tag;
this.seenFor = false;
} else {
tag = 'RELATION';
if (this.value() === '!') {
poppedToken = this.tokens.pop();
tokenData.invert = (ref7 = (ref8 = poppedToken.data) != null ? ref8.original : void 0) != null ? ref7 : poppedToken[1];
}
}
}
} else if (tag === 'IDENTIFIER' && this.seenFor && id === 'from' && isForFrom(prev)) {
tag = 'FORFROM';
this.seenFor = false;
// Throw an error on attempts to use `get` or `set` as keywords, or
// what CoffeeScript would normally interpret as calls to functions named
// `get` or `set`, i.e. `get({foo: function () {}})`.
} else if (tag === 'PROPERTY' && prev) {
if (prev.spaced && (ref9 = prev[0], indexOf.call(CALLABLE, ref9) >= 0) && /^[gs]et$/.test(prev[1]) && this.tokens.length > 1 && ((ref10 = this.tokens[this.tokens.length - 2][0]) !== '.' && ref10 !== '?.' && ref10 !== '@')) {
this.error(`'${prev[1]}' cannot be used as a keyword, or as a function call without parentheses`, prev[2]);
} else if (prev[0] === '.' && this.tokens.length > 1 && (prevprev = this.tokens[this.tokens.length - 2])[0] === 'UNARY' && prevprev[1] === 'new') {
prevprev[0] = 'NEW_TARGET';
} else if (prev[0] === '.' && this.tokens.length > 1 && (prevprev = this.tokens[this.tokens.length - 2])[0] === 'IMPORT' && prevprev[1] === 'import') {
this.seenImport = false;
prevprev[0] = 'IMPORT_META';
} else if (this.tokens.length > 2) {
prevprev = this.tokens[this.tokens.length - 2];
if (((ref11 = prev[0]) === '@' || ref11 === 'THIS') && prevprev && prevprev.spaced && /^[gs]et$/.test(prevprev[1]) && ((ref12 = this.tokens[this.tokens.length - 3][0]) !== '.' && ref12 !== '?.' && ref12 !== '@')) {
this.error(`'${prevprev[1]}' cannot be used as a keyword, or as a function call without parentheses`, prevprev[2]);
}
}
}
if (tag === 'IDENTIFIER' && indexOf.call(RESERVED, id) >= 0 && !inJSXTag) {
this.error(`reserved word '${id}'`, {
length: id.length
});
}
if (!(tag === 'PROPERTY' || this.exportSpecifierList || this.importSpecifierList)) {
if (indexOf.call(COFFEE_ALIASES, id) >= 0) {
alias = id;
id = COFFEE_ALIAS_MAP[id];
tokenData.original = alias;
}
tag = (function() {
switch (id) {
case '!':
return 'UNARY';
case '==':
case '!=':
return 'COMPARE';
case 'true':
case 'false':
return 'BOOL';
case 'break':
case 'continue':
case 'debugger':
return 'STATEMENT';
case '&&':
case '||':
return id;
default:
return tag;
}
})();
}
tagToken = this.token(tag, id, {
length: idLength,
data: tokenData
});
if (alias) {
tagToken.origin = [tag, alias, tagToken[2]];
}
if (poppedToken) {
[tagToken[2].first_line, tagToken[2].first_column, tagToken[2].range[0]] = [poppedToken[2].first_line, poppedToken[2].first_column, poppedToken[2].range[0]];
}
if (colon) {
colonOffset = input.lastIndexOf(inJSXTag ? '=' : ':');
colonToken = this.token(':', ':', {
offset: colonOffset
});
if (inJSXTag) { // used by rewriter
colonToken.jsxColon = true;
}
}
if (inJSXTag && tag === 'IDENTIFIER' && prev[0] !== ':') {
this.token(',', ',', {
length: 0,
origin: tagToken,
generated: true
});
}
return input.length;
}
// Matches numbers, including decimals, hex, and exponential notation.
// Be careful not to interfere with ranges in progress.
numberToken() {
var lexedLength, match, number, parsedValue, tag, tokenData;
if (!(match = NUMBER.exec(this.chunk))) {
return 0;
}
number = match[0];
lexedLength = number.length;
switch (false) {
case !/^0[BOX]/.test(number):
this.error(`radix prefix in '${number}' must be lowercase`, {
offset: 1
});
break;
case !/^(?!0x).*E/.test(number):
this.error(`exponential notation in '${number}' must be indicated with a lowercase 'e'`, {
offset: number.indexOf('E')
});
break;
case !/^0\d*[89]/.test(number):
this.error(`decimal literal '${number}' must not be prefixed with '0'`, {
length: lexedLength
});
break;
case !/^0\d+/.test(number):
this.error(`octal literal '${number}' must be prefixed with '0o'`, {
length: lexedLength
});
}
parsedValue = parseNumber(number);
tokenData = {parsedValue};
tag = parsedValue === 2e308 ? 'INFINITY' : 'NUMBER';
if (tag === 'INFINITY') {
tokenData.original = number;
}
this.token(tag, number, {
length: lexedLength,
data: tokenData
});
return lexedLength;
}
// Matches strings, including multiline strings, as well as heredocs, with or without
// interpolation.
stringToken() {
var attempt, delimiter, doc, end, heredoc, i, indent, match, prev, quote, ref, regex, token, tokens;
[quote] = STRING_START.exec(this.chunk) || [];
if (!quote) {
return 0;
}
// If the preceding token is `from` and this is an import or export statement,
// properly tag the `from`.
prev = this.prev();
if (prev && this.value() === 'from' && (this.seenImport || this.seenExport)) {
prev[0] = 'FROM';
}
regex = (function() {
switch (quote) {
case "'":
return STRING_SINGLE;
case '"':
return STRING_DOUBLE;
case "'''":
return HEREDOC_SINGLE;
case '"""':
return HEREDOC_DOUBLE;
}
})();
({
tokens,
index: end
} = this.matchWithInterpolations(regex, quote));
heredoc = quote.length === 3;
if (heredoc) {
// Find the smallest indentation. It will be removed from all lines later.
indent = null;
doc = ((function() {
var j, len, results;
results = [];
for (i = j = 0, len = tokens.length; j < len; i = ++j) {
token = tokens[i];
if (token[0] === 'NEOSTRING') {
results.push(token[1]);
}
}
return results;
})()).join('#{}');
while (match = HEREDOC_INDENT.exec(doc)) {
attempt = match[1];
if (indent === null || (0 < (ref = attempt.length) && ref < indent.length)) {
indent = attempt;
}
}
}
delimiter = quote.charAt(0);
this.mergeInterpolationTokens(tokens, {
quote,
indent,
endOffset: end
}, (value) => {
return this.validateUnicodeCodePointEscapes(value, {
delimiter: quote
});
});
if (this.atJSXTag()) {
this.token(',', ',', {
length: 0,
origin: this.prev,
generated: true
});
}
return end;
}
// Matches and consumes comments. The comments are taken out of the token
// stream and saved for later, to be reinserted into the output after
// everything has been parsed and the JavaScript code generated.
commentToken(chunk = this.chunk, {heregex, returnCommentTokens = false, offsetInChunk = 0} = {}) {
var commentAttachment, commentAttachments, commentWithSurroundingWhitespace, content, contents, getIndentSize, hasSeenFirstCommentLine, hereComment, hereLeadingWhitespace, hereTrailingWhitespace, i, indentSize, leadingNewline, leadingNewlineOffset, leadingNewlines, leadingWhitespace, length, lineComment, match, matchIllegal, noIndent, nonInitial, placeholderToken, precededByBlankLine, precedingNonCommentLines, prev;
if (!(match = chunk.match(COMMENT))) {
return 0;
}
[commentWithSurroundingWhitespace, hereLeadingWhitespace, hereComment, hereTrailingWhitespace, lineComment] = match;
contents = null;
// Does this comment follow code on the same line?
leadingNewline = /^\s*\n+\s*#/.test(commentWithSurroundingWhitespace);
if (hereComment) {
matchIllegal = HERECOMMENT_ILLEGAL.exec(hereComment);
if (matchIllegal) {
this.error(`block comments cannot contain ${matchIllegal[0]}`, {
offset: '###'.length + matchIllegal.index,
length: matchIllegal[0].length
});
}
// Parse indentation or outdentation as if this block comment didn’t exist.
chunk = chunk.replace(`###${hereComment}###`, '');
// Remove leading newlines, like `Rewriter::removeLeadingNewlines`, to
// avoid the creation of unwanted `TERMINATOR` tokens.
chunk = chunk.replace(/^\n+/, '');
this.lineToken({chunk});
// Pull out the ###-style comment’s content, and format it.
content = hereComment;
contents = [
{
content,
length: commentWithSurroundingWhitespace.length - hereLeadingWhitespace.length - hereTrailingWhitespace.length,
leadingWhitespace: hereLeadingWhitespace
}
];
} else {
// The `COMMENT` regex captures successive line comments as one token.
// Remove any leading newlines before the first comment, but preserve
// blank lines between line comments.
leadingNewlines = '';
content = lineComment.replace(/^(\n*)/, function(leading) {
leadingNewlines = leading;
return '';
});
precedingNonCommentLines = '';
hasSeenFirstCommentLine = false;
contents = content.split('\n').map(function(line, index) {
var comment, leadingWhitespace;
if (!(line.indexOf('#') > -1)) {
precedingNonCommentLines += `\n${line}`;
return;
}
leadingWhitespace = '';
content = line.replace(/^([ |\t]*)#/, function(_, whitespace) {
leadingWhitespace = whitespace;
return '';
});
comment = {
content,
length: '#'.length + content.length,
leadingWhitespace: `${!hasSeenFirstCommentLine ? leadingNewlines : ''}${precedingNonCommentLines}${leadingWhitespace}`,
precededByBlankLine: !!precedingNonCommentLines
};
hasSeenFirstCommentLine = true;
precedingNonCommentLines = '';
return comment;
}).filter(function(comment) {
return comment;
});
}
getIndentSize = function({leadingWhitespace, nonInitial}) {
var lastNewlineIndex;
lastNewlineIndex = leadingWhitespace.lastIndexOf('\n');
if ((hereComment != null) || !nonInitial) {
if (!(lastNewlineIndex > -1)) {
return null;
}
} else {
if (lastNewlineIndex == null) {
lastNewlineIndex = -1;
}
}
return leadingWhitespace.length - 1 - lastNewlineIndex;
};
commentAttachments = (function() {
var j, len, results;
results = [];
for (i = j = 0, len = contents.length; j < len; i = ++j) {
({content, length, leadingWhitespace, precededByBlankLine} = contents[i]);
nonInitial = i !== 0;
leadingNewlineOffset = nonInitial ? 1 : 0;
offsetInChunk += leadingNewlineOffset + leadingWhitespace.length;
indentSize = getIndentSize({leadingWhitespace, nonInitial});
noIndent = (indentSize == null) || indentSize === -1;
commentAttachment = {
content,
here: hereComment != null,
newLine: leadingNewline || nonInitial, // Line comments after the first one start new lines, by definition.
locationData: this.makeLocationData({offsetInChunk, length}),
precededByBlankLine,
indentSize,
indented: !noIndent && indentSize > this.indent,
outdented: !noIndent && indentSize < this.indent
};
if (heregex) {
commentAttachment.heregex = true;
}
offsetInChunk += length;
results.push(commentAttachment);
}
return results;
}).call(this);
prev = this.prev();
if (!prev) {
// If there’s no previous token, create a placeholder token to attach
// this comment to; and follow with a newline.
commentAttachments[0].newLine = true;
this.lineToken({
chunk: this.chunk.slice(commentWithSurroundingWhitespace.length),
offset: commentWithSurroundingWhitespace.length // Set the indent.
});
placeholderToken = this.makeToken('JS', '', {
offset: commentWithSurroundingWhitespace.length,
generated: true
});
placeholderToken.comments = commentAttachments;
this.tokens.push(placeholderToken);
this.newlineToken(commentWithSurroundingWhitespace.length);
} else {
attachCommentsToNode(commentAttachments, prev);
}
if (returnCommentTokens) {
return commentAttachments;
}
return commentWithSurroundingWhitespace.length;
}
// Matches JavaScript interpolated directly into the source via backticks.
jsToken() {
var length, match, matchedHere, script;
if (!(this.chunk.charAt(0) === '`' && (match = (matchedHere = HERE_JSTOKEN.exec(this.chunk)) || JSTOKEN.exec(this.chunk)))) {
return 0;
}
// Convert escaped backticks to backticks, and escaped backslashes
// just before escaped backticks to backslashes
script = match[1];
({length} = match[0]);
this.token('JS', script, {
length,
data: {
here: !!matchedHere
}
});
return length;
}
// Matches regular expression literals, as well as multiline extended ones.
// Lexing regular expressions is difficult to distinguish from division, so we
// borrow some basic heuristics from JavaScript and Ruby.
regexToken() {
var body, closed, comment, commentIndex, commentOpts, commentTokens, comments, delimiter, end, flags, fullMatch, index, leadingWhitespace, match, matchedComment, origin, prev, ref, ref1, regex, tokens;
switch (false) {
case !(match = REGEX_ILLEGAL.exec(this.chunk)):
this.error(`regular expressions cannot begin with ${match[2]}`, {
offset: match.index + match[1].length
});
break;
case !(match = this.matchWithInterpolations(HEREGEX, '///')):
({tokens, index} = match);
comments = [];
while (matchedComment = HEREGEX_COMMENT.exec(this.chunk.slice(0, index))) {
({
index: commentIndex
} = matchedComment);
[fullMatch, leadingWhitespace, comment] = matchedComment;
comments.push({
comment,
offsetInChunk: commentIndex + leadingWhitespace.length
});
}
commentTokens = flatten((function() {
var j, len, results;
results = [];
for (j = 0, len = comments.length; j < len; j++) {
commentOpts = comments[j];
results.push(this.commentToken(commentOpts.comment, Object.assign(commentOpts, {
heregex: true,
returnCommentTokens: true
})));
}
return results;
}).call(this));
break;
case !(match = REGEX.exec(this.chunk)):
[regex, body, closed] = match;
this.validateEscapes(body, {
isRegex: true,
offsetInChunk: 1
});
index = regex.length;
prev = this.prev();
if (prev) {
if (prev.spaced && (ref = prev[0], indexOf.call(CALLABLE, ref) >= 0)) {
if (!closed || POSSIBLY_DIVISION.test(regex)) {
return 0;
}
} else if (ref1 = prev[0], indexOf.call(NOT_REGEX, ref1) >= 0) {
return 0;
}
}
if (!closed) {
this.error('missing / (unclosed regex)');
}
break;
default:
return 0;
}
[flags] = REGEX_FLAGS.exec(this.chunk.slice(index));
end = index + flags.length;
origin = this.makeToken('REGEX', null, {
length: end
});
switch (false) {
case !!VALID_FLAGS.test(flags):
this.error(`invalid regular expression flags ${flags}`, {
offset: index,
length: flags.length
});
break;
case !(regex || tokens.length === 1):
delimiter = body ? '/' : '///';
if (body == null) {
body = tokens[0][1];
}
this.validateUnicodeCodePointEscapes(body, {delimiter});
this.token('REGEX', `/${body}/${flags}`, {
length: end,
origin,
data: {delimiter}
});
break;
default:
this.token('REGEX_START', '(', {
length: 0,
origin,
generated: true
});
this.token('IDENTIFIER', 'RegExp', {
length: 0,
generated: true
});
this.token('CALL_START', '(', {
length: 0,
generated: true
});
this.mergeInterpolationTokens(tokens, {
double: true,
heregex: {flags},
endOffset: end - flags.length,
quote: '///'
}, (str) => {
return this.validateUnicodeCodePointEscapes(str, {delimiter});
});
if (flags) {
this.token(',', ',', {
offset: index - 1,
length: 0,
generated: true
});
this.token('STRING', '"' + flags + '"', {
offset: index,
length: flags.length
});
}
this.token(')', ')', {
offset: end,
length: 0,
generated: true
});
this.token('REGEX_END', ')', {
offset: end,
length: 0,
generated: true
});
}
// Explicitly attach any heregex comments to the REGEX/REGEX_END token.
if (commentTokens != null ? commentTokens.length : void 0) {
addTokenData(this.tokens[this.tokens.length - 1], {
heregexCommentTokens: commentTokens
});
}
return end;
}
// Matches newlines, indents, and outdents, and determines which is which.
// If we can detect that the current line is continued onto the next line,
// then the newline is suppressed:
// elements
// .each( ... )
// .map( ... )
// Keeps track of the level of indentation, because a single outdent token
// can close multiple indents, so we need to know how far in we happen to be.
lineToken({chunk = this.chunk, offset = 0} = {}) {
var backslash, diff, endsContinuationLineIndentation, indent, match, minLiteralLength, newIndentLiteral, noNewlines, prev, ref, size;
if (!(match = MULTI_DENT.exec(chunk))) {
return 0;
}
indent = match[0];
prev = this.prev();
backslash = (prev != null ? prev[0] : void 0) === '\\';
if (!((backslash || ((ref = this.seenFor) != null ? ref.endsLength : void 0) < this.ends.length) && this.seenFor)) {
this.seenFor = false;
}
if (!((backslash && this.seenImport) || this.importSpecifierList)) {
this.seenImport = false;
}
if (!((backslash && this.seenExport) || this.exportSpecifierList)) {
this.seenExport = false;
}
size = indent.length - 1 - indent.lastIndexOf('\n');
noNewlines = this.unfinished();
newIndentLiteral = size > 0 ? indent.slice(-size) : '';
if (!/^(.?)\1*$/.exec(newIndentLiteral)) {
this.error('mixed indentation', {
offset: indent.length
});
return indent.length;
}
minLiteralLength = Math.min(newIndentLiteral.length, this.indentLiteral.length);
if (newIndentLiteral.slice(0, minLiteralLength) !== this.indentLiteral.slice(0, minLiteralLength)) {
this.error('indentation mismatch', {
offset: indent.length
});
return indent.length;
}
if (size - this.continuationLineAdditionalIndent === this.indent) {
if (noNewlines) {
this.suppressNewlines();
} else {
this.newlineToken(offset);
}
return indent.length;
}
if (size > this.indent) {
if (noNewlines) {
if (!backslash) {
this.continuationLineAdditionalIndent = size - this.indent;
}
if (this.continuationLineAdditionalIndent) {
prev.continuationLineIndent = this.indent + this.continuationLineAdditionalIndent;
}
this.suppressNewlines();
return indent.length;
}
if (!this.tokens.length) {
this.baseIndent = this.indent = size;
this.indentLiteral = newIndentLiteral;
return indent.length;
}
diff = size - this.indent + this.outdebt;
this.token('INDENT', diff, {
offset: offset + indent.length - size,
length: size
});
this.indents.push(diff);
this.ends.push({
tag: 'OUTDENT'
});
this.outdebt = this.continuationLineAdditionalIndent = 0;
this.indent = size;
this.indentLiteral = newIndentLiteral;
} else if (size < this.baseIndent) {
this.error('missing indentation', {
offset: offset + indent.length
});
} else {
endsContinuationLineIndentation = this.continuationLineAdditionalIndent > 0;
this.continuationLineAdditionalIndent = 0;
this.outdentToken({
moveOut: this.indent - size,
noNewlines,
outdentLength: indent.length,
offset,
indentSize: size,
endsContinuationLineIndentation
});
}
return indent.length;
}
// Record an outdent token or multiple tokens, if we happen to be moving back
// inwards past several recorded indents. Sets new @indent value.
outdentToken({moveOut, noNewlines, outdentLength = 0, offset = 0, indentSize, endsContinuationLineIndentation}) {
var decreasedIndent, dent, lastIndent, ref, terminatorToken;
decreasedIndent = this.indent - moveOut;
while (moveOut > 0) {
lastIndent = this.indents[this.indents.length - 1];
if (!lastIndent) {
this.outdebt = moveOut = 0;
} else if (this.outdebt && moveOut <= this.outdebt) {
this.outdebt -= moveOut;
moveOut = 0;
} else {
dent = this.indents.pop() + this.outdebt;
if (outdentLength && (ref = this.chunk[outdentLength], indexOf.call(INDENTABLE_CLOSERS, ref) >= 0)) {
decreasedIndent -= dent - moveOut;
moveOut = dent;
}
this.outdebt = 0;
// pair might call outdentToken, so preserve decreasedIndent
this.pair('OUTDENT');
this.token('OUTDENT', moveOut, {
length: outdentLength,
indentSize: indentSize + moveOut - dent
});
moveOut -= dent;
}
}
if (dent) {
this.outdebt -= moveOut;
}
this.suppressSemicolons();
if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
terminatorToken = this.token('TERMINATOR', '\n', {
offset: offset + outdentLength,
length: 0
});
if (endsContinuationLineIndentation) {
terminatorToken.endsContinuationLineIndentation = {
preContinuationLineIndent: this.indent
};
}
}
this.indent = decreasedIndent;
this.indentLiteral = this.indentLiteral.slice(0, decreasedIndent);
return this;
}
// Matches and consumes non-meaningful whitespace. Tag the previous token
// as being “spaced”, because there are some cases where it makes a difference.
whitespaceToken() {
var match, nline, prev;
if (!((match = WHITESPACE.exec(this.chunk)) || (nline = this.chunk.charAt(0) === '\n'))) {
return 0;
}
prev = this.prev();
if (prev) {
prev[match ? 'spaced' : 'newLine'] = true;
}
if (match) {
return match[0].length;
} else {
return 0;
}
}
// Generate a newline token. Consecutive newlines get merged together.
newlineToken(offset) {
this.suppressSemicolons();
if (this.tag() !== 'TERMINATOR') {
this.token('TERMINATOR', '\n', {
offset,
length: 0
});
}
return this;
}
// Use a `\` at a line-ending to suppress the newline.
// The slash is removed here once its job is done.
suppressNewlines() {
var prev;
prev = this.prev();
if (prev[1] === '\\') {
if (prev.comments && this.tokens.length > 1) {
// `@tokens.length` should be at least 2 (some code, then `\`).
// If something puts a `\` after nothing, they deserve to lose any
// comments that trail it.
attachCommentsToNode(prev.comments, this.tokens[this.tokens.length - 2]);
}
this.tokens.pop();
}
return this;
}
jsxToken() {
var afterTag, end, endToken, firstChar, fullId, fullTagName, id, input, j, jsxTag, len, match, offset, openingTagToken, prev, prevChar, properties, property, ref, tagToken, token, tokens;
firstChar = this.chunk[0];
// Check the previous token to detect if attribute is spread.
prevChar = this.tokens.length > 0 ? this.tokens[this.tokens.length - 1][0] : '';
if (firstChar === '<') {
match = JSX_IDENTIFIER.exec(this.chunk.slice(1)) || JSX_FRAGMENT_IDENTIFIER.exec(this.chunk.slice(1));
// Not the right hand side of an unspaced comparison (i.e. `a<b`).
if (!(match && (this.jsxDepth > 0 || !(prev = this.prev()) || prev.spaced || (ref = prev[0], indexOf.call(COMPARABLE_LEFT_SIDE, ref) < 0)))) {
return 0;
}
[input, id] = match;
fullId = id;
if (indexOf.call(id, '.') >= 0) {
[id, ...properties] = id.split('.');
} else {
properties = [];
}
tagToken = this.token('JSX_TAG', id, {
length: id.length + 1,
data: {
openingBracketToken: this.makeToken('<', '<'),
tagNameToken: this.makeToken('IDENTIFIER', id, {
offset: 1
})
}
});
offset = id.length + 1;
for (j = 0, len = properties.length; j < len; j++) {
property = properties[j];
this.token('.', '.', {offset});
offset += 1;
this.token('PROPERTY', property, {offset});
offset += property.length;
}
this.token('CALL_START', '(', {
generated: true
});
this.token('[', '[', {
generated: true
});
this.ends.push({
tag: '/>',
origin: tagToken,
name: id,
properties
});
this.jsxDepth++;
return fullId.length + 1;
} else if (jsxTag = this.atJSXTag()) {
if (this.chunk.slice(0, 2) === '/>') { // Self-closing tag.
this.pair('/>');
this.token(']', ']', {
length: 2,
generated: true
});
this.token('CALL_END', ')', {
length: 2,
generated: true,
data: {
selfClosingSlashToken: this.makeToken('/', '/'),
closingBracketToken: this.makeToken('>', '>', {
offset: 1
})
}
});
this.jsxDepth--;
return 2;
} else if (firstChar === '{') {
if (prevChar === ':') {
// This token represents the start of a JSX attribute value
// that’s an expression (e.g. the `{b}` in `<div a={b} />`).
// Our grammar represents the beginnings of expressions as `(`
// tokens, so make this into a `(` token that displays as `{`.
token = this.token('(', '{');
this.jsxObjAttribute[this.jsxDepth] = false;
// tag attribute name as JSX
addTokenData(this.tokens[this.tokens.length - 3], {
jsx: true
});
} else {
token = this.token('{', '{');
this.jsxObjAttribute[this.jsxDepth] = true;
}
this.ends.push({
tag: '}',
origin: token
});
return 1;
} else if (firstChar === '>') { // end of opening tag
({
// Ignore terminators inside a tag.
origin: openingTagToken
} = this.pair('/>')); // As if the current tag was self-closing.
this.token(']', ']', {
generated: true,
data: {
closingBracketToken: this.makeToken('>', '>')
}
});
this.token(',', 'JSX_COMMA', {
generated: true
});
({
tokens,
index: end
} = this.matchWithInterpolations(INSIDE_JSX, '>', '</', JSX_INTERPOLATION));
this.mergeInterpolationTokens(tokens, {
endOffset: end,
jsx: true
}, (value) => {
return this.validateUnicodeCodePointEscapes(value, {
delimiter: '>'
});
});
match = JSX_IDENTIFIER.exec(this.chunk.slice(end)) || JSX_FRAGMENT_IDENTIFIER.exec(this.chunk.slice(end));
if (!match || match[1] !== `${jsxTag.name}${((function() {
var k, len1, ref1, results;
ref1 = jsxTag.properties;
results = [];
for (k = 0, len1 = ref1.length; k < len1; k++) {
property = ref1[k];
results.push(`.${property}`);
}
return results;
})()).join('')}`) {
this.error(`expected corresponding JSX closing tag for ${jsxTag.name}`, jsxTag.origin.data.tagNameToken[2]);
}
[, fullTagName] = match;
afterTag = end + fullTagName.length;
if (this.chunk[afterTag] !== '>') {
this.error("missing closing > after tag name", {
offset: afterTag,
length: 1
});
}
// -2/+2 for the opening `</` and +1 for the closing `>`.
endToken = this.token('CALL_END', ')', {
offset: end - 2,
length: fullTagName.length + 3,
generated: true,
data: {
closingTagOpeningBracketToken: this.makeToken('<', '<', {
offset: end - 2
}),
closingTagSlashToken: this.makeToken('/', '/', {
offset: end - 1
}),
// TODO: individual tokens for complex tag name? eg < / A . B >
closingTagNameToken: this.makeToken('IDENTIFIER', fullTagName, {
offset: end
}),
closingTagClosingBracketToken: this.makeToken('>', '>', {
offset: end + fullTagName.length
})
}
});
// make the closing tag location data more easily accessible to the grammar
addTokenData(openingTagToken, endToken.data);
this.jsxDepth--;
return afterTag + 1;
} else {
return 0;
}
} else if (this.atJSXTag(1)) {
if (firstChar === '}') {
this.pair(firstChar);
if (this.jsxObjAttribute[this.jsxDepth]) {
this.token('}', '}');
this.jsxObjAttribute[this.jsxDepth] = false;
} else {
this.token(')', '}');
}
this.token(',', ',', {
generated: true
});
return 1;
} else {
return 0;
}
} else {
return 0;
}
}
atJSXTag(depth = 0) {
var i, last, ref;
if (this.jsxDepth === 0) {
return false;
}
i = this.ends.length - 1;
while (((ref = this.ends[i]) != null ? ref.tag : void 0) === 'OUTDENT' || depth-- > 0) { // Ignore indents.
i--;
}
last = this.ends[i];
return (last != null ? last.tag : void 0) === '/>' && last;
}
// We treat all other single characters as a token. E.g.: `( ) , . !`
// Multi-character operators are also literal tokens, so that Jison can assign
// the proper order of operations. There are some symbols that we tag specially
// here. `;` and newlines are both treated as a `TERMINATOR`, we distinguish
// parentheses that indicate a method call from regular parentheses, and so on.
literalToken() {
var match, message, origin, prev, ref, ref1, ref2, ref3, ref4, ref5, skipToken, tag, token, value;
if (match = OPERATOR.exec(this.chunk)) {
[value] = match;
if (CODE.test(value)) {
this.tagParameters();
}
} else {
value = this.chunk.charAt(0);
}
tag = value;
prev = this.prev();
if (prev && indexOf.call(['=', ...COMPOUND_ASSIGN], value) >= 0) {
skipToken = false;
if (value === '=' && ((ref = prev[1]) === '||' || ref === '&&') && !prev.spaced) {
prev[0] = 'COMPOUND_ASSIGN';
prev[1] += '=';
if ((ref1 = prev.data) != null ? ref1.original : void 0) {
prev.data.original += '=';
}
prev[2].range = [prev[2].range[0], prev[2].range[1] + 1];
prev[2].last_column += 1;
prev[2].last_column_exclusive += 1;
prev = this.tokens[this.tokens.length - 2];
skipToken = true;
}
if (prev && prev[0] !== 'PROPERTY') {
origin = (ref2 = prev.origin) != null ? ref2 : prev;
message = isUnassignable(prev[1], origin[1]);
if (message) {
this.error(message, origin[2]);
}
}
if (skipToken) {
return value.length;
}
}
if (value === '(' && (prev != null ? prev[0] : void 0) === 'IMPORT') {
prev[0] = 'DYNAMIC_IMPORT';
}
if (value === '{' && this.seenImport) {
this.importSpecifierList = true;
} else if (this.importSpecifierList && value === '}') {
this.importSpecifierList = false;
} else if (value === '{' && (prev != null ? prev[0] : void 0) === 'EXPORT') {
this.exportSpecifierList = true;
} else if (this.exportSpecifierList && value === '}') {
this.exportSpecifierList = false;
}
if (value === ';') {
if (ref3 = prev != null ? prev[0] : void 0, indexOf.call(['=', ...UNFINISHED], ref3) >= 0) {
this.error('unexpected ;');
}
this.seenFor = this.seenImport = this.seenExport = false;
tag = 'TERMINATOR';
} else if (value === '*' && (prev != null ? prev[0] : void 0) === 'EXPORT') {
tag = 'EXPORT_ALL';
} else if (indexOf.call(MATH, value) >= 0) {
tag = 'MATH';
} else if (indexOf.call(COMPARE, value) >= 0) {
tag = 'COMPARE';
} else if (indexOf.call(COMPOUND_ASSIGN, value) >= 0) {
tag = 'COMPOUND_ASSIGN';
} else if (indexOf.call(UNARY, value) >= 0) {
tag = 'UNARY';
} else if (indexOf.call(UNARY_MATH, value) >= 0) {
tag = 'UNARY_MATH';
} else if (indexOf.call(SHIFT, value) >= 0) {
tag = 'SHIFT';
} else if (value === '?' && (prev != null ? prev.spaced : void 0)) {
tag = 'BIN?';
} else if (prev) {
if (value === '(' && !prev.spaced && (ref4 = prev[0], indexOf.call(CALLABLE, ref4) >= 0)) {
if (prev[0] === '?') {
prev[0] = 'FUNC_EXIST';
}
tag = 'CALL_START';
} else if (value === '[' && (((ref5 = prev[0], indexOf.call(INDEXABLE, ref5) >= 0) && !prev.spaced) || (prev[0] === '::'))) { // `.prototype` can’t be a method you can call.
tag = 'INDEX_START';
switch (prev[0]) {
case '?':
prev[0] = 'INDEX_SOAK';
}
}
}
token = this.makeToken(tag, value);
switch (value) {
case '(':
case '{':
case '[':
this.ends.push({
tag: INVERSES[value],
origin: token
});
break;
case ')':
case '}':
case ']':
this.pair(value);
}
this.tokens.push(this.makeToken(tag, value));
return value.length;
}
// Token Manipulators
// ------------------
// A source of ambiguity in our grammar used to be parameter lists in function
// definitions versus argument lists in function calls. Walk backwards, tagging
// parameters specially in order to make things easier for the parser.
tagParameters() {
var i, paramEndToken, stack, tok, tokens;
if (this.tag() !== ')') {
return this.tagDoIife();
}
stack = [];
({tokens} = this);
i = tokens.length;
paramEndToken = tokens[--i];
paramEndToken[0] = 'PARAM_END';
while (tok = tokens[--i]) {
switch (tok[0]) {
case ')':
stack.push(tok);
break;
case '(':
case 'CALL_START':
if (stack.length) {
stack.pop();
} else if (tok[0] === '(') {
tok[0] = 'PARAM_START';
return this.tagDoIife(i - 1);
} else {
paramEndToken[0] = 'CALL_END';
return this;
}
}
}
return this;
}
// Tag `do` followed by a function differently than `do` followed by eg an
// identifier to allow for different grammar precedence
tagDoIife(tokenIndex) {
var tok;
tok = this.tokens[tokenIndex != null ? tokenIndex : this.tokens.length - 1];
if ((tok != null ? tok[0] : void 0) !== 'DO') {
return this;
}
tok[0] = 'DO_IIFE';
return this;
}
// Close up all remaining open blocks at the end of the file.
closeIndentation() {
return this.outdentToken({
moveOut: this.indent,
indentSize: 0
});
}
// Match the contents of a delimited token and expand variables and expressions
// inside it using Ruby-like notation for substitution of arbitrary
// expressions.