@jkearl/pratt/build/tokenize.js

Pratt parser builder (along with simple tokenizer)

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const matchRegexOperators = /[|\\{}()[\]^$+*?.-]/g;
const selectRegexInner = /^\/(.*)\/.*$/;
const asRegexString = (str) => typeof str === "string"
    ? str.replace(matchRegexOperators, "\\$&")
    : str.toString().match(selectRegexInner)[1];
class Tokenizer {
    constructor(matchers) {
        this.matchers = matchers.map(matcher => ({
            id: "id" in matcher ? matcher.id : matcher.pattern,
            pattern: new RegExp(asRegexString(matcher.pattern)),
        }));
        this.matchAll = new RegExp(matchers.map(matcher => asRegexString(matcher.pattern)).join("|"), "g");
        this.tokenize = this.tokenize.bind(this);
    }
    tokenize(str) {
        const tokens = str.match(this.matchAll);
        if (!tokens) {
            throw new Error("Could not tokenize");
        }
        return tokens.map(value => ({
            ...this.matchers.find(matcher => matcher.pattern.test(value)),
            value,
        }));
    }
}
exports.Tokenizer = Tokenizer;
//# sourceMappingURL=tokenize.js.map