@ayanaware/bentocord/build/commands/internal/Tokenizer.js

Bentocord is a Bento plugin designed to rapidly build fully functional Discord Bots.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Tokenizer = exports.TokenType = void 0;
var TokenType;
(function (TokenType) {
    TokenType[TokenType["WHITESPACE"] = 0] = "WHITESPACE";
    TokenType[TokenType["WORD"] = 1] = "WORD";
    TokenType[TokenType["QUOTE_OPEN"] = 2] = "QUOTE_OPEN";
    TokenType[TokenType["QUOTE_CLOSE"] = 3] = "QUOTE_CLOSE";
    TokenType[TokenType["OPTION"] = 4] = "OPTION";
    TokenType[TokenType["EOF"] = 5] = "EOF";
})(TokenType = exports.TokenType || (exports.TokenType = {}));
class Tokenizer {
    constructor(content) {
        this.tokens = [];
        this.position = 0;
        this.quoteState = false;
        this.content = content;
    }
    tokenize() {
        while (this.content && this.position < this.content.length)
            this.nextToken();
        this.addToken(TokenType.EOF, null);
        return this.tokens;
    }
    addToken(type, value) {
        this.tokens.push({ type, value });
    }
    shift(n) {
        this.position += n;
    }
    addAndShiftToken(type, value) {
        this.addToken(type, value);
        this.shift(value.length);
    }
    match(r) {
        return r.exec(this.content.slice(this.position));
    }
    nextToken() {
        for (const fn of [this.findWhitespace.bind(this), this.findOption.bind(this), this.findQuote.bind(this), this.findWord.bind(this)]) {
            if (fn())
                break;
        }
    }
    findWhitespace() {
        const ws = this.match(/^\s+/);
        if (!ws)
            return false;
        this.addAndShiftToken(TokenType.WHITESPACE, ws[0]);
        return true;
    }
    findOption() {
        // ignore options in quotes
        if (this.quoteState)
            return false;
        // options are words followed by : or =
        const option = this.match(/^([a-z0-9]+)[:=]\s?[^/]/);
        if (!option)
            return false;
        this.addAndShiftToken(TokenType.OPTION, option[0].slice(0, -1));
        return true;
    }
    findQuote() {
        const quote = this.match(/^["“”]/);
        if (!quote)
            return false;
        // Toggle Quote state
        this.addAndShiftToken(this.quoteState ? TokenType.QUOTE_CLOSE : TokenType.QUOTE_OPEN, '"');
        this.quoteState = !this.quoteState;
        return true;
    }
    findWord() {
        const word = this.match(this.quoteState ? /^[^\s"“”]+/ : /^\S+/);
        if (!word)
            return false;
        this.addAndShiftToken(TokenType.WORD, word[0]);
        return true;
    }
}
exports.Tokenizer = Tokenizer;
//# sourceMappingURL=Tokenizer.js.map