jora
Version:
JavaScript object query engine
359 lines (298 loc) • 11 kB
JavaScript
;
const tokenizerTolerantTokenPairs = require('./tokenizer-tolerant-token-pairs.cjs');
const tokens = require('./tokens.cjs');
// Predefined regex for efficient identifier reading
const COMMENT_RX = /\/\/.*?(?:\n|\r\n?|\u2028|\u2029|$)|\/\*(?:.|\s)*?(?:\*\/|$)/y;
const WHITESPACE_RX = /\s+/y;
const REST_TOKENS_RX = new RegExp(Object.keys(tokens.STR_TO_TOKEN).map(s => s.replace(/[\[\]{}()^$|?*+.]/g, '\\$&')).join('|'), 'y');
const IDENTIFIER_RX = /(?:(?:[a-zA-Z_]|\\u[0-9a-fA-F]{4})(?:[a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})*)\b/y;
const KEYWORD_RX = /(?:has no|not in|and|or|not|has|is|in|no|(?:asc|desc)(?:NA?|AN?)?)\b/y;
const REGEXP_RX = /\/(?:\\.|[^/])+\/[gimsu]*/y;
const NUMBER_RX = /(?:[_\d]*\.)?[_\d]+(?:[eE][-+]?[_\d]+)?\b|0[xX][_0-9a-fA-F]+\b/y;
// RegExps are relaxed to allow broken strings; strings are validated during conversion into literals.
// This provides better error messages, as it allows locating the exact problem position.
const STRING_RX = /"(?:\\.|[^"])*"|'(?:\\.|[^'])*'/sy;
const TEMPLATE_START_RX = /`(?:\\.|[^`\\$]|\$(?!\{))*(\$\{|`)/sy;
const TEMPLATE_CONT_END_RX = /\}(?:\\.|[^`\\$]|\$(?!\{))*(\$\{|`)/sy;
const NOOP = () => {};
// Literal value lookup
const LITERALS = new Map([
['true', true],
['false', false],
['null', null],
['undefined', undefined],
['Infinity', Infinity],
['NaN', NaN]
]);
class Token {
#input;
#value;
constructor(type, start, end, input, value = null) {
this.type = type;
this.start = start;
this.end = end;
this.#input = input;
this.#value = value;
}
get name() {
return tokens.tokenNames[this.type];
}
get value() {
return this.#value ?? (this.#value = this.#input.slice(this.start, this.end));
}
}
function* tokenize(input, tolerant = false) {
const nextToken = createTokenizer(input, tolerant);
let token;
while (token = nextToken()) {
yield token;
}
}
function tokenizeSync(input, tolerant = false) {
const nextToken = createTokenizer(input, tolerant);
const tokens = [];
let token;
while (token = nextToken()) {
tokens.push(token);
}
return tokens;
}
function createTokenizer(input, options) {
const {
tolerant: tolerantMode = false,
onComment: onCommentHandler
} = typeof options === 'boolean'
? { tolerant: options }
: options || {};
const onComment = typeof onCommentHandler === 'function'
? onCommentHandler
: Array.isArray(onCommentHandler)
? (start, end) => onCommentHandler.push([start, end])
: NOOP;
const length = input.length;
let pos = 0;
let done = false;
let bracketStack = [];
let preventPrimitive = false;
let preventKeyword = false;
let pendingToken = null;
let prevTokenType = null;
let prevTokenEnd = 0;
function advance(count = 1) {
pos = Math.min(pos + count, length);
}
function match(regexp) {
regexp.lastIndex = pos;
const match = regexp.exec(input);
if (match) {
const value = match[0];
advance(value.length);
return value;
}
return null;
}
function peek() {
return pos < length ? input[pos] : '';
}
function peekNext() {
const index = pos + 1;
return index < length ? input[index] : '';
}
function isIdentStart(ch) {
return (
(ch >= 'A' && ch <= 'Z') || // A-Z
(ch >= 'a' && ch <= 'z') || // a-z
(ch === '_') || /* _ */
(ch === '\\' && peekNext() === 'u')
);
}
function isNumberStart(ch) {
if (ch >= '0' && ch <= '9') { // 0-9
return true;
}
if (ch === '.') {
const next = peekNext();
return (next >= '0' && next <= '9') || next === '_';
}
return false;
}
function trackBracketBalance(tokenType) {
if (tokens.BALANCE_TOKEN_PAIR[tokenType] !== -1) {
bracketStack.push(tokens.BALANCE_TOKEN_PAIR[tokenType]);
} else if (bracketBalanceTop() === tokenType) {
bracketStack.pop();
}
}
function bracketBalanceTop() {
return bracketStack.length > 0
? bracketStack[bracketStack.length - 1]
: -1;
}
// Tolerant mode tokenization wrapper
function nextTokenTolerant(token) {
// Check if we need to insert an empty IDENT in tolerant mode
const shouldInsertEmptyIdent = tokenizerTolerantTokenPairs.TOLERANT_TOKEN_PAIRS.get(prevTokenType)?.[token.type];
if (shouldInsertEmptyIdent) {
// Store new token as pending
pendingToken = token;
// Determine empty IDENT range based on surrounding token types:
// - After keyword or before keyword: zero-width range
// - Otherwise (after dot/operator, before operator/EOF): spans the gap
let emptyStart;
let emptyEnd;
if (prevTokenType === null) {
// First injected ident (no previous token): starts at input beginning
emptyStart = 0;
emptyEnd = tokens.KEYWORD_TOKENS[token.type]
? Math.max(0, token.start - 1)
: token.start;
} else if (tokens.KEYWORD_TOKENS[prevTokenType]) {
// Previous is a keyword: zero-width at next token
emptyStart = token.start;
emptyEnd = token.start;
} else if (tokens.KEYWORD_TOKENS[token.type]) {
// Next token is a keyword: span from previous token's end
// to just before the keyword (absorb whitespace gap)
emptyStart = prevTokenEnd;
emptyEnd = Math.max(prevTokenEnd, token.start - 1);
} else {
// Normal case: span from previous token's end to next token's start
emptyStart = prevTokenEnd;
emptyEnd = token.start;
}
return new Token(tokens.TOKEN_IDENT, emptyStart, emptyEnd, input, '');
}
// Normal token processing
prevTokenType = token.type;
prevTokenEnd = token.end;
return token;
}
// Strict mode tokenization
function nextTokenStrict() {
if (pos >= length) {
done = true;
return tokens.TOKEN_EOF;
}
const start = pos;
const ch = input[pos];
// Numbers
if (isNumberStart(ch) && match(NUMBER_RX)) {
return tokens.TOKEN_NUMBER;
}
// Strings
if ((ch === '"' || ch === "'") && match(STRING_RX)) {
return tokens.TOKEN_STRING;
}
// Template literal/template start
if (ch === '`' && match(TEMPLATE_START_RX)) {
return input[pos - 1] == '`'
? tokens.TOKEN_TEMPLATE
: tokens.TOKEN_TPL_START;
}
// Template continuation/end
if (ch === '}' && bracketBalanceTop() === tokens.TOKEN_TPL_END && match(TEMPLATE_CONT_END_RX)) {
return input[pos - 1] === '`'
? tokens.TOKEN_TPL_END
: tokens.TOKEN_TPL_CONTINUE;
}
// Regular expressions (context-aware using state)
if (ch === '/') {
// If preventPrimitive state is active, treat as division
if (!preventPrimitive && match(REGEXP_RX)) {
return tokens.TOKEN_REGEXP;
}
// Treat as division if it doesn't look like regex
advance();
return tokens.TOKEN_DIVIDE;
}
// Identifiers and method calls
if (isIdentStart(ch)) {
// Keywords
if (!preventKeyword && match(KEYWORD_RX)) {
return tokens.KEYWORDS[input.slice(start, pos)];
}
if (match(IDENTIFIER_RX)) {
// Check for literals
if (!preventKeyword && LITERALS.has(input.slice(start, pos))) {
return tokens.TOKEN_LITERAL;
}
// Check for method call
if (peek() === '(') {
advance(); // consume the (
return tokens.TOKEN_METHOD_OPEN;
}
return tokens.TOKEN_IDENT;
}
}
// Variable references and special symbols
if (ch === '$') {
advance();
if (peek() === '$') {
advance();
return tokens.TOKEN_$$;
}
if (match(IDENTIFIER_RX)) {
// Check for $method(
if (peek() === '(') {
advance(); // consume the (
return tokens.TOKEN_$METHOD_OPEN;
}
return tokens.TOKEN_$IDENT;
}
return tokens.TOKEN_$;
}
// Rest tokens check
if (match(REST_TOKENS_RX)) {
return tokens.STR_TO_TOKEN[input.slice(start, pos)];
}
pos++;
return tokens.TOKEN_BAD;
}
function scanComment() {
const start = pos;
// Both single-line and multi-line comments are starting with '/',
// so we can check for it first before running the more expensive regex match.
// This optimization is especially beneficial because comments are rare in typical queries,
// but checking each position with regex would be costly.
if (peek() === '/' && match(COMMENT_RX)) {
onComment(start, pos, input);
return true;
}
return false;
}
return function nextToken() {
// Check for pending token first
if (pendingToken) {
// Get and clear pending token
const token = pendingToken;
pendingToken = null;
prevTokenType = token.type;
prevTokenEnd = token.end;
return token;
}
if (done) {
return null;
}
// Skip whitespace and comments first
do {
if (match(WHITESPACE_RX)) {
preventKeyword = false;
}
} while (scanComment());
// Read next token
const start = pos;
const nextTokenType = nextTokenStrict();
const token = new Token(nextTokenType, start, pos, input);
preventPrimitive = tokens.PREVENT_PRIMITIVE[nextTokenType];
preventKeyword = tokens.PREVENT_KEYWORD[nextTokenType];
trackBracketBalance(nextTokenType);
return tolerantMode
? nextTokenTolerant(token)
: token;
};
}
exports.LITERALS = LITERALS;
exports.Token = Token;
exports.createTokenizer = createTokenizer;
exports.tokenize = tokenize;
exports.tokenizeSync = tokenizeSync;