@bizone-ai/monaco-json-transform/dist/dstProviders/tokenizeLine.js

JSON Transform language tokenizer (and syntax highlight), hover provider and more

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenType = void 0;
exports.default = tokenizeLine;
const json_transform_utils_1 = require("@bizone-ai/json-transform-utils");
const FunctionContextRegExp = /##([a-z]+[a-z_\d]*)(((\.(?![-\w$]+\()[-\w$]+)|(\[[^\]\n]+]))+|(?=[^\w.]|$))/g;
const FindCommentsRegex = /"(\/\/|\$comment)":\s*"(\\"|[^"])*"/g;
const NumberRegexp = /^-?\d+(\.\d+)?$/;
var TokenType;
(function (TokenType) {
    TokenType[TokenType["VARIABLE"] = 0] = "VARIABLE";
    TokenType[TokenType["VARIABLE_DEPRECATED"] = 1] = "VARIABLE_DEPRECATED";
    TokenType[TokenType["MEMBER"] = 2] = "MEMBER";
    TokenType[TokenType["CONTEXT"] = 3] = "CONTEXT";
    TokenType[TokenType["ANNOTATION"] = 4] = "ANNOTATION";
    TokenType[TokenType["FUNCTION"] = 5] = "FUNCTION";
    TokenType[TokenType["FUNCTION_CONTEXT"] = 6] = "FUNCTION_CONTEXT";
    TokenType[TokenType["FUNCTION_DEPRECATED"] = 7] = "FUNCTION_DEPRECATED";
    TokenType[TokenType["COMMENT"] = 8] = "COMMENT";
    TokenType[TokenType["STRING"] = 9] = "STRING";
    TokenType[TokenType["NUMBER"] = 10] = "NUMBER";
    TokenType[TokenType["KEYWORD"] = 11] = "KEYWORD";
    TokenType[TokenType["NO_STYLE"] = 12] = "NO_STYLE";
})(TokenType || (exports.TokenType = TokenType = {}));
var TokenModifier;
(function (TokenModifier) {
    TokenModifier[TokenModifier["DECLARATION"] = 0] = "DECLARATION";
})(TokenModifier || (TokenModifier = {}));
function tokenizeLine(line, lineNumber, ts) {
    // FUNCTIONS (name and args symbols)
    const funcMatches = json_transform_utils_1.functionsParser.matchAllFunctionsInLine(line);
    for (const match of funcMatches) {
        const func = json_transform_utils_1.functionsParser.get(match.name);
        const deprecated = func?.deprecated;
        ts.tokens.push({
            line: lineNumber,
            char: match.index,
            length: match.keyLength,
            type: deprecated ? TokenType.FUNCTION_DEPRECATED : TokenType.FUNCTION,
            modifier: TokenModifier.DECLARATION,
        });
        // :
        if (match.input?.index) {
            ts.tokens.push({
                line: lineNumber,
                char: match.input.index - 1,
                length: 1,
                type: TokenType.NO_STYLE,
                modifier: TokenModifier.DECLARATION,
            });
        }
        // arguments
        if (match.args) {
            for (const arg of match.args) {
                if (line[arg.index] !== "'") {
                    // let strings stay strings (so other tokens will override it)
                    ts.tokens.push({
                        line: lineNumber,
                        char: arg.index,
                        length: arg.length,
                        type: arg.value === "true" || arg.value === "false"
                            ? TokenType.KEYWORD
                            : arg.value && NumberRegexp.test(arg.value)
                                ? TokenType.NUMBER
                                : TokenType.NO_STYLE,
                        modifier: TokenModifier.DECLARATION,
                    });
                }
            }
        }
    }
    // FUNCTION CONTEXT (##current)
    let iter = line.matchAll(FunctionContextRegExp);
    for (let iterResult = iter.next(), match = iterResult.value; !iterResult.done; iterResult = iter.next(), match = iterResult.value) {
        if (!match || typeof match.index === "undefined")
            continue;
        ts.tokens.push({
            line: lineNumber,
            char: match.index,
            length: match[0].length,
            type: TokenType.FUNCTION_CONTEXT,
            modifier: TokenModifier.DECLARATION,
        });
    }
    // VARIABLES ($. #...)
    iter = line.matchAll(json_transform_utils_1.transformUtils.variableDetectionRegExp);
    for (let iterResult = iter.next(), match = iterResult.value; !iterResult.done; iterResult = iter.next(), match = iterResult.value) {
        if (!match || typeof match.index === "undefined")
            continue;
        let type = TokenType.VARIABLE;
        const matchLength = match[0].length, modifier = TokenModifier.DECLARATION;
        let skip = false;
        if (json_transform_utils_1.transformUtils.matchesAnyOfContextVariables(match[0])) {
            type = TokenType.CONTEXT;
        }
        else if (json_transform_utils_1.transformUtils.matchesAnyOfSpecialKeys(match[0])) {
            type = TokenType.MEMBER;
        }
        else if (json_transform_utils_1.transformUtils.matchesAnyOfAdditionalContext(match[0])) {
            type = TokenType.ANNOTATION;
        }
        else if (match?.[0][0] === "#") {
            // not one of the context variables
            skip = true;
        }
        if (!skip) {
            ts.tokens.push({
                line: lineNumber,
                char: match.index,
                length: matchLength,
                type,
                modifier,
            });
        }
    }
    // COMMENTS
    iter = line.matchAll(FindCommentsRegex);
    for (let iterResult = iter.next(), match = iterResult.value; !iterResult.done; iterResult = iter.next(), match = iterResult.value) {
        if (!match || typeof match.index === "undefined")
            continue;
        ts.tokens.push({
            line: lineNumber,
            char: match.index,
            length: match[0].length,
            type: TokenType.COMMENT,
            modifier: TokenModifier.DECLARATION,
        });
    }
}
//# sourceMappingURL=tokenizeLine.js.map