prettier-plugin-asciidoc/parser/lexer/tokenize.js

Format AsciiDoc files with prettier 📖

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenize = exports.AllTokens = void 0;
const chevrotain_1 = require("chevrotain");
const attribute_list_tokens_1 = require("./attribute-list-tokens");
const shared_token_1 = require("./shared-token");
const string_tokens_1 = require("./string-tokens");
const rootTokens = [
    shared_token_1.Headline,
    attribute_list_tokens_1.AttributeListStart,
    shared_token_1.Space,
    shared_token_1.EmptyLine,
    shared_token_1.Newline,
    shared_token_1.Tab,
    shared_token_1.InlineText,
];
const modes = {
    root: rootTokens,
    [attribute_list_tokens_1.AttributeListMode]: attribute_list_tokens_1.AttributeListTokens,
    [string_tokens_1.StringMode]: string_tokens_1.StringTokens,
};
const lexer = new chevrotain_1.Lexer({
    defaultMode: "root",
    modes,
}, { positionTracking: "full" });
const nonUniqueTokens = Object.values(modes).flat();
exports.AllTokens = Array.from(new Set(nonUniqueTokens).values());
function tokenize(text) {
    return lexer.tokenize(text).tokens;
}
exports.tokenize = tokenize;