chevrotain
Version:
Chevrotain is a high performance fault tolerant javascript parsing DSL for building recursive decent parsers
640 lines • 33.8 kB
JavaScript
import { analyzeTokenTypes, charCodeToOptimizedIndex, cloneEmptyGroups, DEFAULT_MODE, LineTerminatorOptimizedTester, performRuntimeChecks, performWarningRuntimeChecks, validatePatterns, } from "./lexer.js";
import { PRINT_WARNING, timer, toFastProperties } from "@chevrotain/utils";
import { augmentTokenTypes } from "./tokens.js";
import { defaultLexerErrorProvider } from "./lexer_errors_public.js";
import { clearRegExpParserCache } from "./reg_exp_parser.js";
export var LexerDefinitionErrorType;
(function (LexerDefinitionErrorType) {
LexerDefinitionErrorType[LexerDefinitionErrorType["MISSING_PATTERN"] = 0] = "MISSING_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_PATTERN"] = 1] = "INVALID_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["EOI_ANCHOR_FOUND"] = 2] = "EOI_ANCHOR_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["UNSUPPORTED_FLAGS_FOUND"] = 3] = "UNSUPPORTED_FLAGS_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["DUPLICATE_PATTERNS_FOUND"] = 4] = "DUPLICATE_PATTERNS_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_GROUP_TYPE_FOUND"] = 5] = "INVALID_GROUP_TYPE_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["PUSH_MODE_DOES_NOT_EXIST"] = 6] = "PUSH_MODE_DOES_NOT_EXIST";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE"] = 7] = "MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY"] = 8] = "MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST"] = 9] = "MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST";
LexerDefinitionErrorType[LexerDefinitionErrorType["LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED"] = 10] = "LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED";
LexerDefinitionErrorType[LexerDefinitionErrorType["SOI_ANCHOR_FOUND"] = 11] = "SOI_ANCHOR_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["EMPTY_MATCH_PATTERN"] = 12] = "EMPTY_MATCH_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["NO_LINE_BREAKS_FLAGS"] = 13] = "NO_LINE_BREAKS_FLAGS";
LexerDefinitionErrorType[LexerDefinitionErrorType["UNREACHABLE_PATTERN"] = 14] = "UNREACHABLE_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["IDENTIFY_TERMINATOR"] = 15] = "IDENTIFY_TERMINATOR";
LexerDefinitionErrorType[LexerDefinitionErrorType["CUSTOM_LINE_BREAK"] = 16] = "CUSTOM_LINE_BREAK";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE"] = 17] = "MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE";
})(LexerDefinitionErrorType || (LexerDefinitionErrorType = {}));
const DEFAULT_LEXER_CONFIG = {
deferDefinitionErrorsHandling: false,
positionTracking: "full",
lineTerminatorsPattern: /\n|\r\n?/g,
lineTerminatorCharacters: ["\n", "\r"],
ensureOptimizations: false,
safeMode: false,
errorMessageProvider: defaultLexerErrorProvider,
traceInitPerf: false,
skipValidations: false,
recoveryEnabled: true,
};
Object.freeze(DEFAULT_LEXER_CONFIG);
export class Lexer {
constructor(lexerDefinition, config = DEFAULT_LEXER_CONFIG) {
this.lexerDefinition = lexerDefinition;
this.lexerDefinitionErrors = [];
this.lexerDefinitionWarning = [];
this.patternIdxToConfig = {};
this.charCodeToPatternIdxToConfig = {};
this.modes = [];
this.emptyGroups = {};
this.trackStartLines = true;
this.trackEndLines = true;
this.hasCustom = false;
this.canModeBeOptimized = {};
// Duplicated from the parser's perf trace trait to allow future extraction
// of the lexer to a separate package.
this.TRACE_INIT = (phaseDesc, phaseImpl) => {
// No need to optimize this using NOOP pattern because
// It is not called in a hot spot...
if (this.traceInitPerf === true) {
this.traceInitIndent++;
const indent = new Array(this.traceInitIndent + 1).join("\t");
if (this.traceInitIndent < this.traceInitMaxIdent) {
console.log(`${indent}--> <${phaseDesc}>`);
}
const { time, value } = timer(phaseImpl);
/* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
const traceMethod = time > 10 ? console.warn : console.log;
if (this.traceInitIndent < this.traceInitMaxIdent) {
traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`);
}
this.traceInitIndent--;
return value;
}
else {
return phaseImpl();
}
};
if (typeof config === "boolean") {
throw Error("The second argument to the Lexer constructor is now an ILexerConfig Object.\n" +
"a boolean 2nd argument is no longer supported");
}
this.config = Object.assign({}, DEFAULT_LEXER_CONFIG, config);
const traceInitVal = this.config.traceInitPerf;
if (traceInitVal === true) {
this.traceInitMaxIdent = Infinity;
this.traceInitPerf = true;
}
else if (typeof traceInitVal === "number") {
this.traceInitMaxIdent = traceInitVal;
this.traceInitPerf = true;
}
this.traceInitIndent = -1;
this.TRACE_INIT("Lexer Constructor", () => {
let actualDefinition;
let hasOnlySingleMode = true;
this.TRACE_INIT("Lexer Config handling", () => {
if (this.config.lineTerminatorsPattern ===
DEFAULT_LEXER_CONFIG.lineTerminatorsPattern) {
// optimized built-in implementation for the defaults definition of lineTerminators
this.config.lineTerminatorsPattern = LineTerminatorOptimizedTester;
}
else {
if (this.config.lineTerminatorCharacters ===
DEFAULT_LEXER_CONFIG.lineTerminatorCharacters) {
throw Error("Error: Missing <lineTerminatorCharacters> property on the Lexer config.\n" +
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS");
}
}
if (config.safeMode && config.ensureOptimizations) {
throw Error('"safeMode" and "ensureOptimizations" flags are mutually exclusive.');
}
this.trackStartLines = /full|onlyStart/i.test(this.config.positionTracking);
this.trackEndLines = /full/i.test(this.config.positionTracking);
// Convert SingleModeLexerDefinition into a IMultiModeLexerDefinition.
if (Array.isArray(lexerDefinition)) {
actualDefinition = {
modes: { defaultMode: [...lexerDefinition] },
defaultMode: DEFAULT_MODE,
};
}
else {
// no conversion needed, input should already be a IMultiModeLexerDefinition
hasOnlySingleMode = false;
actualDefinition = Object.assign({}, lexerDefinition);
}
});
if (this.config.skipValidations === false) {
this.TRACE_INIT("performRuntimeChecks", () => {
this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(performRuntimeChecks(actualDefinition, this.trackStartLines, this.config.lineTerminatorCharacters));
});
this.TRACE_INIT("performWarningRuntimeChecks", () => {
this.lexerDefinitionWarning = this.lexerDefinitionWarning.concat(performWarningRuntimeChecks(actualDefinition, this.trackStartLines, this.config.lineTerminatorCharacters));
});
}
// for extra robustness to avoid throwing a none informative error message
actualDefinition.modes = actualDefinition.modes
? actualDefinition.modes
: {};
// an error of undefined TokenTypes will be detected in "performRuntimeChecks" above.
// this transformation is to increase robustness in the case of partially invalid lexer definition.
Object.entries(actualDefinition.modes).forEach(([currModeName, currModeValue]) => {
actualDefinition.modes[currModeName] = currModeValue.filter((currTokType) => currTokType !== undefined);
});
const allModeNames = Object.keys(actualDefinition.modes);
Object.entries(actualDefinition.modes).forEach(([currModName, currModDef]) => {
this.TRACE_INIT(`Mode: <${currModName}> processing`, () => {
this.modes.push(currModName);
if (this.config.skipValidations === false) {
this.TRACE_INIT(`validatePatterns`, () => {
this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(validatePatterns(currModDef, allModeNames));
});
}
// If definition errors were encountered, the analysis phase may fail unexpectedly/
// Considering a lexer with definition errors may never be used, there is no point
// to performing the analysis anyhow...
if (this.lexerDefinitionErrors.length === 0) {
augmentTokenTypes(currModDef);
let currAnalyzeResult;
this.TRACE_INIT(`analyzeTokenTypes`, () => {
currAnalyzeResult = analyzeTokenTypes(currModDef, {
lineTerminatorCharacters: this.config.lineTerminatorCharacters,
positionTracking: config.positionTracking,
ensureOptimizations: config.ensureOptimizations,
safeMode: config.safeMode,
tracer: this.TRACE_INIT,
});
});
this.patternIdxToConfig[currModName] =
currAnalyzeResult.patternIdxToConfig;
this.charCodeToPatternIdxToConfig[currModName] =
currAnalyzeResult.charCodeToPatternIdxToConfig;
this.emptyGroups = Object.assign({}, this.emptyGroups, currAnalyzeResult.emptyGroups);
this.hasCustom = currAnalyzeResult.hasCustom || this.hasCustom;
this.canModeBeOptimized[currModName] =
currAnalyzeResult.canBeOptimized;
}
});
});
this.defaultMode = actualDefinition.defaultMode;
if (this.lexerDefinitionErrors.length > 0 &&
!this.config.deferDefinitionErrorsHandling) {
const allErrMessages = this.lexerDefinitionErrors.map((error) => {
return error.message;
});
const allErrMessagesString = allErrMessages.join("-----------------------\n");
throw new Error("Errors detected in definition of Lexer:\n" + allErrMessagesString);
}
// Only print warning if there are no errors, This will avoid pl
this.lexerDefinitionWarning.forEach((warningDescriptor) => {
PRINT_WARNING(warningDescriptor.message);
});
this.TRACE_INIT("Choosing sub-methods implementations", () => {
// Choose the relevant internal implementations for this specific parser.
// These implementations should be in-lined by the JavaScript engine
// to provide optimal performance in each scenario.
if (hasOnlySingleMode) {
this.handleModes = () => { };
}
if (this.trackStartLines === false) {
this.computeNewColumn = (x) => x;
}
if (this.trackEndLines === false) {
this.updateTokenEndLineColumnLocation = () => { };
}
if (/full/i.test(this.config.positionTracking)) {
this.createTokenInstance = this.createFullToken;
}
else if (/onlyStart/i.test(this.config.positionTracking)) {
this.createTokenInstance = this.createStartOnlyToken;
}
else if (/onlyOffset/i.test(this.config.positionTracking)) {
this.createTokenInstance = this.createOffsetOnlyToken;
}
else {
throw Error(`Invalid <positionTracking> config option: "${this.config.positionTracking}"`);
}
if (this.hasCustom) {
this.addToken = this.addTokenUsingPush;
this.handlePayload = this.handlePayloadWithCustom;
}
else {
this.addToken = this.addTokenUsingMemberAccess;
this.handlePayload = this.handlePayloadNoCustom;
}
});
this.TRACE_INIT("Failed Optimization Warnings", () => {
const unOptimizedModes = Object.entries(this.canModeBeOptimized).reduce((cannotBeOptimized, [modeName, canBeOptimized]) => {
if (canBeOptimized === false) {
cannotBeOptimized.push(modeName);
}
return cannotBeOptimized;
}, []);
if (config.ensureOptimizations && unOptimizedModes.length > 0) {
throw Error(`Lexer Modes: < ${unOptimizedModes.join(", ")} > cannot be optimized.\n` +
'\t Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode.\n' +
"\t Or inspect the console log for details on how to resolve these issues.");
}
});
this.TRACE_INIT("clearRegExpParserCache", () => {
clearRegExpParserCache();
});
this.TRACE_INIT("toFastProperties", () => {
toFastProperties(this);
});
});
}
tokenize(text, initialMode = this.defaultMode) {
if (this.lexerDefinitionErrors.length > 0) {
const allErrMessages = this.lexerDefinitionErrors.map((error) => {
return error.message;
});
const allErrMessagesString = allErrMessages.join("-----------------------\n");
throw new Error("Unable to Tokenize because Errors detected in definition of Lexer:\n" +
allErrMessagesString);
}
return this.tokenizeInternal(text, initialMode);
}
// There is quite a bit of duplication between this and "tokenizeInternalLazy"
// This is intentional due to performance considerations.
// this method also used quite a bit of `!` none null assertions because it is too optimized
// for `tsc` to always understand it is "safe"
tokenizeInternal(text, initialMode) {
let i, j, k, matchAltImage, longerAlt, matchedImage, payload, altPayload, imageLength, group, tokType, newToken, errLength, msg, match;
const orgText = text;
const orgLength = orgText.length;
let offset = 0;
let matchedTokensIndex = 0;
// initializing the tokensArray to the "guessed" size.
// guessing too little will still reduce the number of array re-sizes on pushes.
// guessing too large (Tested by guessing x4 too large) may cost a bit more of memory
// but would still have a faster runtime by avoiding (All but one) array resizing.
const guessedNumberOfTokens = this.hasCustom
? 0 // will break custom token pattern APIs the matchedTokens array will contain undefined elements.
: Math.floor(text.length / 10);
const matchedTokens = new Array(guessedNumberOfTokens);
const errors = [];
let line = this.trackStartLines ? 1 : undefined;
let column = this.trackStartLines ? 1 : undefined;
const groups = cloneEmptyGroups(this.emptyGroups);
const trackLines = this.trackStartLines;
const lineTerminatorPattern = this.config.lineTerminatorsPattern;
let currModePatternsLength = 0;
let patternIdxToConfig = [];
let currCharCodeToPatternIdxToConfig = [];
const modeStack = [];
const emptyArray = [];
Object.freeze(emptyArray);
let isOptimizedMode = false;
const pop_mode = (popToken) => {
// TODO: perhaps avoid this error in the edge case there is no more input?
if (modeStack.length === 1 &&
// if we have both a POP_MODE and a PUSH_MODE this is in-fact a "transition"
// So no error should occur.
popToken.tokenType.PUSH_MODE === undefined) {
// if we try to pop the last mode there lexer will no longer have ANY mode.
// thus the pop is ignored, an error will be created and the lexer will continue parsing in the previous mode.
const msg = this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(popToken);
errors.push({
offset: popToken.startOffset,
line: popToken.startLine,
column: popToken.startColumn,
length: popToken.image.length,
message: msg,
});
}
else {
modeStack.pop();
const newMode = modeStack.at(-1);
patternIdxToConfig = this.patternIdxToConfig[newMode];
currCharCodeToPatternIdxToConfig =
this.charCodeToPatternIdxToConfig[newMode];
currModePatternsLength = patternIdxToConfig.length;
const modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
isOptimizedMode = true;
}
else {
isOptimizedMode = false;
}
}
};
function push_mode(newMode) {
modeStack.push(newMode);
currCharCodeToPatternIdxToConfig =
this.charCodeToPatternIdxToConfig[newMode];
patternIdxToConfig = this.patternIdxToConfig[newMode];
currModePatternsLength = patternIdxToConfig.length;
currModePatternsLength = patternIdxToConfig.length;
const modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
isOptimizedMode = true;
}
else {
isOptimizedMode = false;
}
}
// this pattern seems to avoid a V8 de-optimization, although that de-optimization does not
// seem to matter performance wise.
push_mode.call(this, initialMode);
let currConfig;
const recoveryEnabled = this.config.recoveryEnabled;
while (offset < orgLength) {
matchedImage = null;
imageLength = -1;
const nextCharCode = orgText.charCodeAt(offset);
let chosenPatternIdxToConfig;
if (isOptimizedMode) {
const optimizedCharIdx = charCodeToOptimizedIndex(nextCharCode);
const possiblePatterns = currCharCodeToPatternIdxToConfig[optimizedCharIdx];
chosenPatternIdxToConfig =
possiblePatterns !== undefined ? possiblePatterns : emptyArray;
}
else {
chosenPatternIdxToConfig = patternIdxToConfig;
}
const chosenPatternsLength = chosenPatternIdxToConfig.length;
for (i = 0; i < chosenPatternsLength; i++) {
currConfig = chosenPatternIdxToConfig[i];
const currPattern = currConfig.pattern;
payload = null;
// manually in-lined because > 600 chars won't be in-lined in V8
const singleCharCode = currConfig.short;
if (singleCharCode !== false) {
if (nextCharCode === singleCharCode) {
// single character string
imageLength = 1;
matchedImage = currPattern;
}
}
else if (currConfig.isCustom === true) {
match = currPattern.exec(orgText, offset, matchedTokens, groups);
if (match !== null) {
matchedImage = match[0];
imageLength = matchedImage.length;
if (match.payload !== undefined) {
payload = match.payload;
}
}
else {
matchedImage = null;
}
}
else {
currPattern.lastIndex = offset;
imageLength = this.matchLength(currPattern, text, offset);
}
// longer alts handling
if (imageLength !== -1) {
// even though this pattern matched we must try a another longer alternative.
// this can be used to prioritize keywords over identifiers
longerAlt = currConfig.longerAlt;
if (longerAlt !== undefined) {
matchedImage = text.substring(offset, offset + imageLength);
const longerAltLength = longerAlt.length;
for (k = 0; k < longerAltLength; k++) {
const longerAltConfig = patternIdxToConfig[longerAlt[k]];
const longerAltPattern = longerAltConfig.pattern;
altPayload = null;
// single Char can never be a longer alt so no need to test it.
// manually in-lined because > 600 chars won't be in-lined in V8
if (longerAltConfig.isCustom === true) {
match = longerAltPattern.exec(orgText, offset, matchedTokens, groups);
if (match !== null) {
matchAltImage = match[0];
if (match.payload !== undefined) {
altPayload = match.payload;
}
}
else {
matchAltImage = null;
}
}
else {
longerAltPattern.lastIndex = offset;
matchAltImage = this.match(longerAltPattern, text, offset);
}
if (matchAltImage && matchAltImage.length > matchedImage.length) {
matchedImage = matchAltImage;
imageLength = matchAltImage.length;
payload = altPayload;
currConfig = longerAltConfig;
// Exit the loop early after matching one of the longer alternatives
// The first matched alternative takes precedence
break;
}
}
}
break;
}
}
// successful match
if (imageLength !== -1) {
group = currConfig.group;
if (group !== undefined) {
matchedImage =
matchedImage !== null
? matchedImage // for custom Tokens we will already have the `matchedImage`
: text.substring(offset, offset + imageLength);
tokType = currConfig.tokenTypeIdx;
newToken = this.createTokenInstance(matchedImage, offset, tokType, currConfig.tokenType, line, column, imageLength);
this.handlePayload(newToken, payload);
if (group === false) {
matchedTokensIndex = this.addToken(matchedTokens, matchedTokensIndex, newToken);
}
else {
groups[group].push(newToken);
}
}
// line terminator handling
if (trackLines === true && currConfig.canLineTerminator === true) {
let numOfLTsInMatch = 0;
let foundTerminator;
let lastLTEndOffset;
lineTerminatorPattern.lastIndex = 0;
do {
// only for skipped tokens the matchedImage may be null at this point
matchedImage =
matchedImage !== null
? matchedImage
: text.substring(offset, offset + imageLength);
foundTerminator = lineTerminatorPattern.test(matchedImage);
if (foundTerminator === true) {
lastLTEndOffset = lineTerminatorPattern.lastIndex - 1;
numOfLTsInMatch++;
}
} while (foundTerminator === true);
if (numOfLTsInMatch !== 0) {
line = line + numOfLTsInMatch;
column = imageLength - lastLTEndOffset;
this.updateTokenEndLineColumnLocation(newToken, group, lastLTEndOffset, numOfLTsInMatch, line, column, imageLength);
}
else {
column = this.computeNewColumn(column, imageLength);
}
}
else {
column = this.computeNewColumn(column, imageLength);
}
offset = offset + imageLength;
// will be NOOP if no modes present
this.handleModes(currConfig, pop_mode, push_mode, newToken);
}
else {
// error recovery, drop characters until we identify a valid token's start point
const errorStartOffset = offset;
const errorLine = line;
const errorColumn = column;
let foundResyncPoint = recoveryEnabled === false;
while (foundResyncPoint === false && offset < orgLength) {
offset++;
for (j = 0; j < currModePatternsLength; j++) {
const currConfig = patternIdxToConfig[j];
const currPattern = currConfig.pattern;
// manually in-lined because > 600 chars won't be in-lined in V8
const singleCharCode = currConfig.short;
if (singleCharCode !== false) {
if (orgText.charCodeAt(offset) === singleCharCode) {
// single character string
foundResyncPoint = true;
}
}
else if (currConfig.isCustom === true) {
foundResyncPoint =
currPattern.exec(orgText, offset, matchedTokens, groups) !== null;
}
else {
currPattern.lastIndex = offset;
foundResyncPoint = currPattern.exec(text) !== null;
}
if (foundResyncPoint === true) {
break;
}
}
}
errLength = offset - errorStartOffset;
column = this.computeNewColumn(column, errLength);
// at this point we either re-synced or reached the end of the input text
msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(orgText, errorStartOffset, errLength, errorLine, errorColumn, modeStack.at(-1));
errors.push({
offset: errorStartOffset,
line: errorLine,
column: errorColumn,
length: errLength,
message: msg,
});
if (recoveryEnabled === false) {
break;
}
}
}
// if we do have custom patterns which push directly into the
// TODO: custom tokens should not push directly??
if (!this.hasCustom) {
// if we guessed a too large size for the tokens array this will shrink it to the right size.
matchedTokens.length = matchedTokensIndex;
}
return {
tokens: matchedTokens,
groups: groups,
errors: errors,
};
}
handleModes(config, pop_mode, push_mode, newToken) {
if (config.pop === true) {
// need to save the PUSH_MODE property as if the mode is popped
// patternIdxToPopMode is updated to reflect the new mode after popping the stack
const pushMode = config.push;
pop_mode(newToken);
if (pushMode !== undefined) {
push_mode.call(this, pushMode);
}
}
else if (config.push !== undefined) {
push_mode.call(this, config.push);
}
}
// TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler
updateTokenEndLineColumnLocation(newToken, group, lastLTIdx, numOfLTsInMatch, line, column, imageLength) {
let lastCharIsLT, fixForEndingInLT;
if (group !== undefined) {
// a none skipped multi line Token, need to update endLine/endColumn
lastCharIsLT = lastLTIdx === imageLength - 1;
fixForEndingInLT = lastCharIsLT ? -1 : 0;
if (!(numOfLTsInMatch === 1 && lastCharIsLT === true)) {
// if a token ends in a LT that last LT only affects the line numbering of following Tokens
newToken.endLine = line + fixForEndingInLT;
// the last LT in a token does not affect the endColumn either as the [columnStart ... columnEnd)
// inclusive to exclusive range.
newToken.endColumn = column - 1 + -fixForEndingInLT;
}
// else single LT in the last character of a token, no need to modify the endLine/EndColumn
}
}
computeNewColumn(oldColumn, imageLength) {
return oldColumn + imageLength;
}
createOffsetOnlyToken(image, startOffset, tokenTypeIdx, tokenType) {
return {
image,
startOffset,
tokenTypeIdx,
tokenType,
};
}
createStartOnlyToken(image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn) {
return {
image,
startOffset,
startLine,
startColumn,
tokenTypeIdx,
tokenType,
};
}
createFullToken(image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn, imageLength) {
return {
image,
startOffset,
endOffset: startOffset + imageLength - 1,
startLine,
endLine: startLine,
startColumn,
endColumn: startColumn + imageLength - 1,
tokenTypeIdx,
tokenType,
};
}
addTokenUsingPush(tokenVector, index, tokenToAdd) {
tokenVector.push(tokenToAdd);
return index;
}
addTokenUsingMemberAccess(tokenVector, index, tokenToAdd) {
tokenVector[index] = tokenToAdd;
index++;
return index;
}
handlePayloadNoCustom(token, payload) { }
handlePayloadWithCustom(token, payload) {
if (payload !== null) {
token.payload = payload;
}
}
match(pattern, text, offset) {
const found = pattern.test(text);
if (found === true) {
return text.substring(offset, pattern.lastIndex);
}
return null;
}
matchLength(pattern, text, offset) {
const found = pattern.test(text);
if (found === true) {
return pattern.lastIndex - offset;
}
return -1;
}
}
Lexer.SKIPPED = "This marks a skipped Token pattern, this means each token identified by it will " +
"be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.";
Lexer.NA = /NOT_APPLICABLE/;
//# sourceMappingURL=lexer_public.js.map