lovelang/dist/tokenizer.js

🖤 LoveLang: A romantic wrapper language over TypeScript/TSX

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenize = tokenize;
const KEYWORDS = [
    "hi babe",
    "this is",
    "remember",
    "babe,",
    "say",
    "when babe thinks",
    "if babe feels",
    "then",
    "else",
    "whenever babe says",
    "class",
    "try",
    "catch",
    "stop thinking",
    "render",
    "component",
];
// Sort keywords in descending order of length to prefer longer matches like "this is" before "this"
const SORTED_KEYWORDS = [...KEYWORDS].sort((a, b) => b.length - a.length);
function tokenize(src) {
    const tokens = [];
    let i = 0;
    while (i < src.length) {
        // Skip whitespace
        if (/\s/.test(src[i])) {
            i++;
            continue;
        }
        // Handle string literals
        if (src[i] === '"') {
            let j = i + 1;
            while (src[j] !== '"' && j < src.length)
                j++;
            if (src[j] !== '"')
                throw new Error("Unterminated string");
            tokens.push({ type: "STRING", value: src.slice(i + 1, j) });
            i = j + 1;
            continue;
        }
        // Handle numbers
        if (/\d/.test(src[i])) {
            let j = i;
            while (/\d/.test(src[j]))
                j++;
            tokens.push({ type: "NUMBER", value: src.slice(i, j) });
            i = j;
            continue;
        }
        // Handle multi-char punct like --
        if (src[i] === "-" && src[i + 1] === "-") {
            tokens.push({ type: "PUNCT", value: "--" });
            i += 2;
            continue;
        }
        // Handle multi-word keywords
        const rest = src.slice(i);
        const kw = SORTED_KEYWORDS.find((k) => rest.startsWith(k));
        if (kw) {
            tokens.push({ type: "KEYWORD", value: kw });
            i += kw.length;
            continue;
        }
        // Handle identifiers (single word tokens)
        if (/[A-Za-z_]/.test(src[i])) {
            let j = i;
            while (/[A-Za-z0-9_]/.test(src[j]))
                j++;
            tokens.push({ type: "IDENT", value: src.slice(i, j) });
            i = j;
            continue;
        }
        // Handle punctuation
        if (/[\=\;\{\}\,\<\>]/.test(src[i])) {
            tokens.push({ type: "PUNCT", value: src[i] });
            i++;
            continue;
        }
        throw new Error(`Unexpected character '${src[i]}'`);
    }
    return tokens;
}