@vectara/vectara-ui
Version:
Vectara's design system, codified as a React and Sass component library
97 lines (96 loc) • 4.01 kB
JavaScript
/**
* This exposes a utility that wraps an ANTLR-generated lexer in a TokensProvider class and returns that class.
* The underlying logic is mostly lifted from https://tomassetti.me/writing-a-browser-based-editor-using-monaco-and-antlr.
*/
import { BaseErrorListener, CharStream, CommonTokenStream } from "antlr4ng";
const EOF = -1;
/**
* Generates a TokensProvider for use with the CodeEditor component.
* Provide the optional syntaxChecker config to enable client-side syntax checking.
*/
export const generateTokensProvider = (Lexer, syntaxChecker) => {
return class TokensProvider {
getInitialState() {
return new MonacoState();
}
tokenize(line) {
if (line.length === 0)
return new MonacoLineTokens([]);
return this.tokensForLine(line);
}
tokensForLine(input) {
const lexerInputStream = CharStream.fromString(input);
const lexer = new Lexer(lexerInputStream);
const myTokens = [];
let done = false;
// If doing client-side syntax checking, create a second input stream for the parser.
// This is because using a single stream for both results in
// the lexer consuming the stream, leaving nothing for the parser to consume.
// TODO: Check if there's a way to use just one. In the interest of time, this will do.
if (syntaxChecker) {
const parserInputStream = CharStream.fromString(input);
const tokenStream = new CommonTokenStream(new Lexer(parserInputStream));
const parser = new syntaxChecker.Parser(tokenStream);
class ConsoleErrorListener extends BaseErrorListener {
syntaxError(recognizer, offendingSymbol, line, column, msg) {
syntaxChecker === null || syntaxChecker === void 0 ? void 0 : syntaxChecker.onError(offendingSymbol, line, column, msg);
}
}
parser.addErrorListener(new ConsoleErrorListener());
parser.parse();
}
do {
const token = lexer.nextToken();
if (token == null) {
done = true;
}
else {
// We exclude EOF
if (token.type == EOF) {
done = true;
}
else if (lexer.literalNames[token.type]) {
// Allow tokenizing of literal names.
// Facilitates colorizing of tokens like "+".
const tokenTypeName = lexer.literalNames[token.type];
if (tokenTypeName) {
const myToken = new MonacoToken("LITERAL", token.column);
myTokens.push(myToken);
}
}
else {
const tokenTypeName = lexer.symbolicNames[token.type];
if (tokenTypeName) {
const myToken = new MonacoToken(tokenTypeName, token.column);
myTokens.push(myToken);
}
}
}
} while (!done);
myTokens.sort((a, b) => (a.startIndex > b.startIndex ? 1 : -1));
return new MonacoLineTokens(myTokens);
}
};
};
class MonacoState {
clone() {
return new MonacoState();
}
// This is hard-coded to true because this implementation exists solely to
// satisfy the monaco.languages.IState interface.
equals() {
return true;
}
}
class MonacoToken {
constructor(ruleName, startIndex) {
this.scopes = ruleName;
this.startIndex = startIndex;
}
}
class MonacoLineTokens {
constructor(tokens) {
this.endState = new MonacoState();
this.tokens = tokens;
}
}