toylang/dist/Tokenizer.js

A toy programming language built with TypeScript for learning purposes

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Tokenizer = exports.TokenTypes = void 0;
var TokenTypes;
(function (TokenTypes) {
    TokenTypes["let"] = "let";
    TokenTypes["if"] = "if";
    TokenTypes["else"] = "else";
    TokenTypes["while"] = "while";
    TokenTypes["do"] = "do";
    TokenTypes["for"] = "for";
    TokenTypes["true"] = "true";
    TokenTypes["false"] = "false";
    TokenTypes["null"] = "null";
    TokenTypes["def"] = "def";
    TokenTypes["return"] = "return";
    TokenTypes["class"] = "class";
    TokenTypes["new"] = "new";
    TokenTypes["this"] = "this";
    TokenTypes["super"] = "super";
    TokenTypes["extends"] = "extends";
    TokenTypes["SEMI"] = "SEMI";
    TokenTypes["CURLY_END"] = "CURLY_END";
    TokenTypes["CURLY_START"] = "CURLY_START";
    TokenTypes["PAREN_START"] = "PAREN_START";
    TokenTypes["PAREN_END"] = "PAREN_END";
    TokenTypes["COMMA"] = "COMMA";
    TokenTypes["DOT"] = "DOT";
    TokenTypes["BRACKET_END"] = "BRACKET_END";
    TokenTypes["BRACKET_START"] = "BRACKET_START";
    TokenTypes["STRING"] = "STRING";
    TokenTypes["NUMBER"] = "NUMBER";
    TokenTypes["IDENTIFIER"] = "IDENTIFIER";
    TokenTypes["EQUALITY_OPERATOR"] = "EQUALITY_OPERATOR";
    TokenTypes["SIMPLE_ASSIGNMENT"] = "SIMPLE_ASSIGNMENT";
    TokenTypes["COMPLEX_ASSIGNMENT"] = "COMPLEX_ASSIGNMENT";
    TokenTypes["ADDITITIVE_OPERATOR"] = "ADDITITIVE_OPERATOR";
    TokenTypes["MULTIPLICATIVE_OPERATOR"] = "MULTIPLICATIVE_OPERATOR";
    TokenTypes["RELATIONAL_OPERATOR"] = "RELATIONAL_OPERATOR";
    TokenTypes["LOGICAL_AND"] = "LOGICAL_AND";
    TokenTypes["LOGICAL_OR"] = "LOGICAL_OR";
    TokenTypes["LOGICAL_NOT"] = "LOGICAL_NOT";
})(TokenTypes = exports.TokenTypes || (exports.TokenTypes = {}));
var spec = [
    [/^\s+/, null],
    [/^\/\/.*/, null],
    [/^\/\*[\s\S]*?\*\//, null],
    // Symbols, Delimiters
    [/^;/, TokenTypes.SEMI],
    [/^\}/, TokenTypes.CURLY_END],
    [/^\{/, TokenTypes.CURLY_START],
    [/^\(/, TokenTypes.PAREN_START],
    [/^\)/, TokenTypes.PAREN_END],
    [/^\,/, TokenTypes.COMMA],
    [/^\./, TokenTypes.DOT],
    [/^\]/, TokenTypes.BRACKET_END],
    [/^\[/, TokenTypes.BRACKET_START],
    // keywords
    [/^\blet\b/, TokenTypes.let],
    [/^\bif\b/, TokenTypes.if],
    [/^\belse\b/, TokenTypes.else],
    [/^\bwhile\b/, TokenTypes.while],
    [/^\bdo\b/, TokenTypes.do],
    [/^\bfor\b/, TokenTypes.for],
    [/^\btrue\b/, TokenTypes.true],
    [/^\bfalse\b/, TokenTypes.false],
    [/^\bnull\b/, TokenTypes.null],
    [/^\bdef\b/, TokenTypes.def],
    [/^\breturn\b/, TokenTypes.return],
    [/^\bclass\b/, TokenTypes.class],
    [/^\bnew\b/, TokenTypes.new],
    [/^\bthis\b/, TokenTypes.this],
    [/^\bsuper\b/, TokenTypes.super],
    [/^\bextends\b/, TokenTypes.extends],
    // Numbers
    [/^\d+/, TokenTypes.NUMBER],
    // Identifiers
    [/^\w+/, TokenTypes.IDENTIFIER],
    // Equality operators
    [/^[=!]=/, TokenTypes.EQUALITY_OPERATOR],
    // Assignment Operators
    [/^=/, TokenTypes.SIMPLE_ASSIGNMENT],
    [/^[\*\/\+\-]=/, TokenTypes.COMPLEX_ASSIGNMENT],
    // Math operators
    [/^[\+\-]/, TokenTypes.ADDITITIVE_OPERATOR],
    [/^[*\/]/, TokenTypes.MULTIPLICATIVE_OPERATOR],
    // relational operators
    [/^[><]=?/, TokenTypes.RELATIONAL_OPERATOR],
    // logical operators
    [/^&&/, TokenTypes.LOGICAL_AND],
    [/^\|\|/, TokenTypes.LOGICAL_OR],
    [/^!/, TokenTypes.LOGICAL_NOT],
    // Strings
    [/^"[^"]*"/, TokenTypes.STRING],
    [/^'[^']*'/, TokenTypes.STRING],
];
var Tokenizer = /** @class */ (function () {
    function Tokenizer() {
    }
    Tokenizer.prototype.init = function (string) {
        this._string = string;
        this._cursor = 0;
    };
    Tokenizer.prototype.hasMoreTokens = function () {
        return this._cursor < this._string.length;
    };
    Tokenizer.prototype.getNextToken = function () {
        if (!this.hasMoreTokens()) {
            return null;
        }
        var string = this._string.slice(this._cursor);
        for (var _i = 0, spec_1 = spec; _i < spec_1.length; _i++) {
            var _a = spec_1[_i], regex = _a[0], type = _a[1];
            var tokenValue = this._match(regex, string);
            // cannot match rule, continue
            if (tokenValue === null) {
                continue;
            }
            // skip token (eg: whitespace)
            if (type === null) {
                return this.getNextToken();
            }
            return {
                type: type,
                value: tokenValue,
            };
        }
        throw new SyntaxError("Unexpected token: \"" + string[0] + "\"");
    };
    Tokenizer.prototype._match = function (regex, string) {
        var matched = regex.exec(string);
        if (matched === null) {
            return null;
        }
        this._cursor += matched[0].length;
        return matched[0];
    };
    return Tokenizer;
}());
exports.Tokenizer = Tokenizer;