unified-query/dist/parsers/scanner.js

Composable search input with autocompletion and a rich query-language parser for the Unified Data System

// src/parse/scanner.ts
import { registry } from '../analyzers/registry.js';
const isIdent = (c) => /[A-Za-z0-9_-]/.test(c);
export const VALID_KEYWORDS = Object.keys(registry);
/**
 * Pass-1 scanner (handles only '\@' escapes).
 *
 * • head   : text before first un-escaped '@'
 * • keyword: '@' IDENT
 * • body   : up to next un-escaped '@' or EOS
 */
export function scan(input) {
    const segments = [];
    const errors = [];
    const seen = new Set(); // track first keyword uses
    let i = 0;
    let segStart = 0;
    const push = (keyword, bodyStart, bodyEnd) => {
        segments.push({
            keyword: keyword,
            tokens: [],
            errors: [],
            body: input.slice(bodyStart, bodyEnd),
            from: segStart,
            to: bodyEnd,
            raw: input.slice(segStart, bodyEnd),
            ignored: false, // default; may flip below
        });
    };
    /* ── HEAD ─────────────────────────────────────────────── */
    while (i < input.length && !(input[i] === '@' && (i === 0 || input[i - 1] !== '\\')))
        i++;
    if (i > 0)
        push('head', segStart, i);
    /* ── KEYWORD BLOCKS ───────────────────────────────────── */
    while (i < input.length) {
        segStart = i; // '@'
        i++; // skip '@'
        // keyword ident
        const kwStart = i;
        while (i < input.length && isIdent(input[i]))
            i++;
        const keyword = input.slice(kwStart, i);
        // stray '@' with no following ident → treat as literal head segment
        // TODO: this is just a patch; should think how to handle empty keywords: "@"
        if (!keyword) {
            segments.push({
                keyword: 'head',
                tokens: [],
                errors: [],
                body: '@',
                from: segStart,
                to: segStart + 1,
                raw: '@',
                ignored: true
            });
            continue; // don't try to parse a real keyword here
        }
        // optional space
        while (i < input.length && input[i] === ' ')
            i++;
        const bodyStart = i;
        // body until next un-escaped '@' or EOS
        while (i < input.length && !(input[i] === '@' && input[i - 1] !== '\\'))
            i++;
        push(keyword, bodyStart, i);
    }
    /* ── Validation pass (unknown / duplicate) ───────────── */
    for (const seg of segments) {
        if (seg.keyword === 'head')
            continue;
        const kwFrom = seg.from + 1; // position of first letter
        const kwTo = kwFrom + seg.keyword.length;
        if (!VALID_KEYWORDS.includes(seg.keyword)) {
            seg.ignored = true;
            addErr(`unknown keyword "@${seg.keyword}"`, kwFrom - 1, kwTo);
        }
        else if (seen.has(seg.keyword)) {
            seg.ignored = true;
            addErr(`duplicate keyword "@${seg.keyword}" ignored`, kwFrom - 1, kwTo);
        }
        else {
            seen.add(seg.keyword);
        }
    }
    return { segments, errors };
    /* helper */
    function addErr(message, from, to) {
        errors.push({ message, token: input.slice(from, to), from, to });
    }
}