jora/cjs/lang/parser/tokenizer-tolerant-token-pairs.cjs

JavaScript object query engine

'use strict';

const tokens = require('./tokens.cjs');

// Optimized tolerant token pairs with shared sets for better memory usage
const keywords = [tokens.TOKEN_AND, tokens.TOKEN_OR, tokens.TOKEN_IN, tokens.TOKEN_NOTIN, tokens.TOKEN_HAS, tokens.TOKEN_HASNO, tokens.TOKEN_IS];
const operators = [
    tokens.TOKEN_PLUS, tokens.TOKEN_MINUS, tokens.TOKEN_MULTIPLY, tokens.TOKEN_DIVIDE, tokens.TOKEN_MODULO, tokens.TOKEN_PIPE,
    tokens.TOKEN_EQUALS, tokens.TOKEN_NOT_EQUALS, tokens.TOKEN_MATCH,
    tokens.TOKEN_GREATER_THAN_EQUALS, tokens.TOKEN_LESS_THAN_EQUALS, tokens.TOKEN_LESS_THAN, tokens.TOKEN_GREATER_THAN
];
const baseTokens = [
    tokens.TOKEN_COMMA, tokens.TOKEN_QUESTION, tokens.TOKEN_COLON, tokens.TOKEN_SEMICOLON, tokens.TOKEN_EOF,
    tokens.TOKEN_CLOSE_BRACKET, tokens.TOKEN_CLOSE_PAREN, tokens.TOKEN_CLOSE_BRACE,
    tokens.TOKEN_TPL_CONTINUE, tokens.TOKEN_TPL_END, tokens.TOKEN_ORDER,
    ...operators,
    ...keywords
];

// Shared set of tokens that can follow a missing identifier
const commonNextTokens = tokens.createTokenSet(...baseTokens);

// Specialized sets for tokens that need exclusions
const emptyBlockNextTokens = tokens.createTokenSet(...baseTokens);
emptyBlockNextTokens[tokens.TOKEN_CLOSE_PAREN] = 0;

const emptyBracketNextTokens = tokens.createTokenSet(...baseTokens);
emptyBracketNextTokens[tokens.TOKEN_CLOSE_BRACKET] = 0;

const dollarNextTokens = tokens.createTokenSet(
    tokens.TOKEN_COMMA, tokens.TOKEN_QUESTION, tokens.TOKEN_COLON,
    tokens.TOKEN_TPL_CONTINUE, tokens.TOKEN_TPL_END, tokens.TOKEN_ORDER,
    ...operators,
    ...keywords
);
// Exclude operators and keywords that work directly with $
[
    tokens.TOKEN_PIPE, tokens.TOKEN_EQUALS, tokens.TOKEN_PLUS, tokens.TOKEN_MINUS,
    tokens.TOKEN_MULTIPLY, tokens.TOKEN_DIVIDE, tokens.TOKEN_MODULO,
    tokens.TOKEN_IN, tokens.TOKEN_HAS, tokens.TOKEN_NOTIN, tokens.TOKEN_HASNO
].forEach(token => dollarNextTokens[token] = 0);

// Create the lookup map with optimized shared sets
const TOLERANT_TOKEN_PAIRS = new Map([
    // Tokens that can precede missing identifiers
    [null, commonNextTokens],
    [tokens.TOKEN_QUESTION, commonNextTokens],
    [tokens.TOKEN_COLON, commonNextTokens],
    [tokens.TOKEN_SEMICOLON, commonNextTokens],
    [tokens.TOKEN_COMMA, commonNextTokens],
    [tokens.TOKEN_DOT, commonNextTokens],
    [tokens.TOKEN_DOT_DOT, commonNextTokens],
    [tokens.TOKEN_OPEN_PAREN, commonNextTokens],
    [tokens.TOKEN_ARROW, commonNextTokens],
    [tokens.TOKEN_NOT, commonNextTokens],
    [tokens.TOKEN_NO, commonNextTokens],

    // Tokens with exclusions for empty constructs
    [tokens.TOKEN_DOT_OPEN_PAREN, emptyBlockNextTokens],
    [tokens.TOKEN_DOT_DOT_OPEN_PAREN, emptyBlockNextTokens],
    [tokens.TOKEN_METHOD_OPEN, emptyBlockNextTokens],
    [tokens.TOKEN_$METHOD_OPEN, emptyBlockNextTokens],
    [tokens.TOKEN_OPEN_BRACKET, emptyBracketNextTokens],
    [tokens.TOKEN_DOT_OPEN_BRACKET, emptyBracketNextTokens],

    // Special handling for $ token
    [tokens.TOKEN_$, dollarNextTokens],

    // Operators and keywords
    ...operators.map(op => [op, commonNextTokens]),
    ...keywords.map(kw => [kw, commonNextTokens])
]);

exports.TOLERANT_TOKEN_PAIRS = TOLERANT_TOKEN_PAIRS;