@codeque/cli
Version:
Multiline code search for every language. Structural code search for JavaScript, TypeScript, HTML and CSS
414 lines (413 loc) • 19.1 kB
JavaScript
;
// Copyright 2014, 2015, 2016, 2017, 2018, 2019, 2020 Simon Lydell
// License: MIT.
//@ts-nocheck
/*eslint-disable*/
let IdentifierName, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens;
RegularExpressionLiteral =
/\/(?![*\/])(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\]\\]).|\\.)*(\/[$_\u200C\u200D\p{ID_Continue}]*|\\)?/uy;
Punctuator =
/--|\+\+|=>|\.{3}|\??\.(?!\d)|(?:&&|\|\||\?\?|[+\-%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2}|\/(?![\/*]))=?|[?~,:;[\](){}]/y;
IdentifierName =
/(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]+\})+/uy;
StringLiteral = /(['"])(?:(?!\1)[^\\\n\r]|\\(?:\r\n|[^]))*(\1)?/y;
NumericLiteral =
/(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[oO][0-7](?:_?[0-7])*|0[bB][01](?:_?[01])*)n?|0n|[1-9](?:_?\d)*n|(?:(?:0(?!\d)|0\d*[89]\d*|[1-9](?:_?\d)*)(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|0[0-7]+/y;
Template = /[`}](?:[^`\\$]|\\[^]|\$(?!\{))*(`|\$\{)?/y;
WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/uy;
LineTerminatorSequence = /\r?\n|[\r\u2028\u2029]/y;
MultiLineComment = /\/\*(?:[^*]|\*(?!\/))*(\*\/)?/y;
SingleLineComment = /\/\/.*/y;
JSXPunctuator = /[<>.:={}]|\/(?![\/*])/y;
JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/uy;
JSXString = /(['"])(?:(?!\1)[^])*(\1)?/y;
JSXText = /[^<>{}]+/y;
TokensPrecedingExpression =
/^(?:[\/+-]|\.{3}|\?(?:InterpolationIn(?:JSX|Template)|NoLineTerminatorHere|NonExpressionParenEnd|UnaryIncDec))?$|[{}([,;<>=*%&|^!~?:]$/;
TokensNotPrecedingObjectLiteral =
/^(?:=>|[;\]){}]|else|\?(?:NoLineTerminatorHere|NonExpressionParenEnd))?$/;
KeywordsWithExpressionAfter =
/^(?:await|case|default|delete|do|else|instanceof|new|return|throw|typeof|void|yield)$/;
KeywordsWithNoLineTerminatorAfter = /^(?:return|throw|yield)$/;
Newline = RegExp(LineTerminatorSequence.source);
module.exports = jsTokens = function* (input, { jsx = false } = {}) {
let braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack;
({ length } = input);
lastIndex = 0;
lastSignificantToken = '';
stack = [{ tag: 'JS' }];
braces = [];
parenNesting = 0;
postfixIncDec = false;
while (lastIndex < length) {
mode = stack[stack.length - 1];
switch (mode.tag) {
case 'JS':
case 'JSNonExpressionParen':
case 'InterpolationInTemplate':
case 'InterpolationInJSX':
if (input[lastIndex] === '/' &&
input[lastIndex - 1] !== '<' &&
(TokensPrecedingExpression.test(lastSignificantToken) ||
KeywordsWithExpressionAfter.test(lastSignificantToken))) {
RegularExpressionLiteral.lastIndex = lastIndex;
if ((match = RegularExpressionLiteral.exec(input))) {
lastIndex = RegularExpressionLiteral.lastIndex;
lastSignificantToken = match[0];
postfixIncDec = true;
yield {
type: 'RegularExpressionLiteral',
value: match[0],
closed: match[1] !== void 0 && match[1] !== '\\'
};
continue;
}
}
Punctuator.lastIndex = lastIndex;
if ((match = Punctuator.exec(input))) {
punctuator = match[0];
nextLastIndex = Punctuator.lastIndex;
nextLastSignificantToken = punctuator;
switch (punctuator) {
case '(':
if (lastSignificantToken === '?NonExpressionParenKeyword') {
stack.push({
tag: 'JSNonExpressionParen',
nesting: parenNesting
});
}
parenNesting++;
postfixIncDec = false;
break;
case ')':
parenNesting--;
postfixIncDec = true;
if (mode.tag === 'JSNonExpressionParen' &&
parenNesting === mode.nesting) {
stack.pop();
nextLastSignificantToken = '?NonExpressionParenEnd';
postfixIncDec = false;
}
break;
case '{':
Punctuator.lastIndex = 0;
isExpression =
!TokensNotPrecedingObjectLiteral.test(lastSignificantToken) &&
(TokensPrecedingExpression.test(lastSignificantToken) ||
KeywordsWithExpressionAfter.test(lastSignificantToken));
braces.push(isExpression);
postfixIncDec = false;
break;
case '}':
switch (mode.tag) {
case 'InterpolationInTemplate':
if (braces.length === mode.nesting) {
Template.lastIndex = lastIndex;
match = Template.exec(input);
lastIndex = Template.lastIndex;
lastSignificantToken = match[0];
if (match[1] === '${') {
lastSignificantToken = '?InterpolationInTemplate';
postfixIncDec = false;
yield {
type: 'TemplateMiddle',
value: match[0]
};
}
else {
stack.pop();
postfixIncDec = true;
yield {
type: 'TemplateTail',
value: match[0],
closed: match[1] === '`'
};
}
continue;
}
break;
case 'InterpolationInJSX':
if (braces.length === mode.nesting) {
stack.pop();
lastIndex += 1;
lastSignificantToken = '}';
yield {
type: 'JSXPunctuator',
value: '}'
};
continue;
}
}
postfixIncDec = braces.pop();
nextLastSignificantToken = postfixIncDec
? '?ExpressionBraceEnd'
: '}';
break;
case ']':
postfixIncDec = true;
break;
case '++':
case '--':
nextLastSignificantToken = postfixIncDec
? '?PostfixIncDec'
: '?UnaryIncDec';
break;
case '<':
if (jsx &&
(TokensPrecedingExpression.test(lastSignificantToken) ||
KeywordsWithExpressionAfter.test(lastSignificantToken))) {
stack.push({ tag: 'JSXTag' });
lastIndex += 1;
lastSignificantToken = '<';
yield {
type: 'JSXPunctuator',
value: punctuator
};
continue;
}
postfixIncDec = false;
break;
default:
postfixIncDec = false;
}
lastIndex = nextLastIndex;
lastSignificantToken = nextLastSignificantToken;
yield {
type: 'Punctuator',
value: punctuator
};
continue;
}
IdentifierName.lastIndex = lastIndex;
if ((match = IdentifierName.exec(input))) {
lastIndex = IdentifierName.lastIndex;
nextLastSignificantToken = match[0];
switch (match[0]) {
case 'for':
case 'if':
case 'while':
case 'with':
if (lastSignificantToken !== '.' &&
lastSignificantToken !== '?.') {
nextLastSignificantToken = '?NonExpressionParenKeyword';
}
}
lastSignificantToken = nextLastSignificantToken;
postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]);
yield {
type: 'IdentifierName',
value: match[0]
};
continue;
}
StringLiteral.lastIndex = lastIndex;
if ((match = StringLiteral.exec(input))) {
lastIndex = StringLiteral.lastIndex;
lastSignificantToken = match[0];
postfixIncDec = true;
yield {
type: 'StringLiteral',
value: match[0],
closed: match[2] !== void 0
};
continue;
}
NumericLiteral.lastIndex = lastIndex;
if ((match = NumericLiteral.exec(input))) {
lastIndex = NumericLiteral.lastIndex;
lastSignificantToken = match[0];
postfixIncDec = true;
yield {
type: 'NumericLiteral',
value: match[0]
};
continue;
}
Template.lastIndex = lastIndex;
if ((match = Template.exec(input))) {
lastIndex = Template.lastIndex;
lastSignificantToken = match[0];
if (match[1] === '${') {
lastSignificantToken = '?InterpolationInTemplate';
stack.push({
tag: 'InterpolationInTemplate',
nesting: braces.length
});
postfixIncDec = false;
yield {
type: 'TemplateHead',
value: match[0]
};
}
else {
postfixIncDec = true;
yield {
type: 'NoSubstitutionTemplate',
value: match[0],
closed: match[1] === '`'
};
}
continue;
}
break;
case 'JSXTag':
case 'JSXTagEnd':
JSXPunctuator.lastIndex = lastIndex;
if ((match = JSXPunctuator.exec(input))) {
lastIndex = JSXPunctuator.lastIndex;
nextLastSignificantToken = match[0];
switch (match[0]) {
case '<':
stack.push({ tag: 'JSXTag' });
break;
case '>':
stack.pop();
if (lastSignificantToken === '/' || mode.tag === 'JSXTagEnd') {
nextLastSignificantToken = '?JSX';
postfixIncDec = true;
}
else {
stack.push({ tag: 'JSXChildren' });
}
break;
case '{':
stack.push({
tag: 'InterpolationInJSX',
nesting: braces.length
});
nextLastSignificantToken = '?InterpolationInJSX';
postfixIncDec = false;
break;
case '/':
if (lastSignificantToken === '<') {
stack.pop();
if (stack[stack.length - 1].tag === 'JSXChildren') {
stack.pop();
}
stack.push({ tag: 'JSXTagEnd' });
}
}
lastSignificantToken = nextLastSignificantToken;
yield {
type: 'JSXPunctuator',
value: match[0]
};
continue;
}
JSXIdentifier.lastIndex = lastIndex;
if ((match = JSXIdentifier.exec(input))) {
lastIndex = JSXIdentifier.lastIndex;
lastSignificantToken = match[0];
yield {
type: 'JSXIdentifier',
value: match[0]
};
continue;
}
JSXString.lastIndex = lastIndex;
if ((match = JSXString.exec(input))) {
lastIndex = JSXString.lastIndex;
lastSignificantToken = match[0];
yield {
type: 'JSXString',
value: match[0],
closed: match[2] !== void 0
};
continue;
}
break;
case 'JSXChildren':
JSXText.lastIndex = lastIndex;
if ((match = JSXText.exec(input))) {
lastIndex = JSXText.lastIndex;
lastSignificantToken = match[0];
yield {
type: 'JSXText',
value: match[0]
};
continue;
}
switch (input[lastIndex]) {
case '<':
stack.push({ tag: 'JSXTag' });
lastIndex++;
lastSignificantToken = '<';
yield {
type: 'JSXPunctuator',
value: '<'
};
continue;
case '{':
stack.push({
tag: 'InterpolationInJSX',
nesting: braces.length
});
lastIndex++;
lastSignificantToken = '?InterpolationInJSX';
postfixIncDec = false;
yield {
type: 'JSXPunctuator',
value: '{'
};
continue;
}
}
WhiteSpace.lastIndex = lastIndex;
if ((match = WhiteSpace.exec(input))) {
lastIndex = WhiteSpace.lastIndex;
yield {
type: 'WhiteSpace',
value: match[0]
};
continue;
}
LineTerminatorSequence.lastIndex = lastIndex;
if ((match = LineTerminatorSequence.exec(input))) {
lastIndex = LineTerminatorSequence.lastIndex;
postfixIncDec = false;
if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
lastSignificantToken = '?NoLineTerminatorHere';
}
yield {
type: 'LineTerminatorSequence',
value: match[0]
};
continue;
}
MultiLineComment.lastIndex = lastIndex;
if ((match = MultiLineComment.exec(input))) {
lastIndex = MultiLineComment.lastIndex;
if (Newline.test(match[0])) {
postfixIncDec = false;
if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
lastSignificantToken = '?NoLineTerminatorHere';
}
}
yield {
type: 'MultiLineComment',
value: match[0],
closed: match[1] !== void 0
};
continue;
}
SingleLineComment.lastIndex = lastIndex;
if ((match = SingleLineComment.exec(input))) {
lastIndex = SingleLineComment.lastIndex;
postfixIncDec = false;
yield {
type: 'SingleLineComment',
value: match[0]
};
continue;
}
firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex));
lastIndex += firstCodePoint.length;
lastSignificantToken = firstCodePoint;
postfixIncDec = false;
yield {
type: mode.tag.startsWith('JSX') ? 'JSXInvalid' : 'Invalid',
value: firstCodePoint
};
}
return void 0;
};