xast/build/Lexer.js

AST parsing library

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Lexer = void 0;
const Token_1 = require("./Token");
const Source_1 = require("./Source");
const TokenKind_1 = require("./TokenKind");
const utils_1 = require("./utils");
const characterClasses_1 = require("./characterClasses");
const createToken_1 = require("./createToken");
class Lexer {
    constructor(source) {
        const sourceObj = Source_1.Source.isSource(source) ? source : new Source_1.Source(source);
        const startOfFileToken = new Token_1.Token(TokenKind_1.TokenKind.SOF, 0, 0, 0, 0);
        this.source = sourceObj;
        this.lastToken = startOfFileToken;
        this.token = startOfFileToken;
        this.line = 1;
        this.lineStart = 0;
        this._tokenizers = new utils_1.MapList();
    }
    get [Symbol.toStringTag]() {
        return 'Lexer';
    }
    add(code, tokenizer) {
        this._tokenizers.set(code, tokenizer);
    }
    tokenize(position, code) {
        const tokenizers = this._tokenizers.get(code) || [];
        for (const tokenizer of tokenizers) {
            const token = tokenizer(this, position, code);
            if (token) {
                return token;
            }
        }
    }
    advance() {
        this.lastToken = this.token;
        const token = (this.token = this.lookahead());
        return token;
    }
    lookahead() {
        let token = this.token;
        if (token.kind !== TokenKind_1.TokenKind.EOF) {
            do {
                if (token.next) {
                    token = token.next;
                }
                else {
                    const nextToken = this.readNextToken(token.end);
                    token.next = nextToken;
                    nextToken.prev = token;
                    token = nextToken;
                }
            } while (token.kind === TokenKind_1.TokenKind.COMMENT);
        }
        return token;
    }
    readNextToken(start) {
        const source = this.source;
        const body = source.body;
        const bodyLength = body.length;
        let position = start;
        while (position < bodyLength) {
            const code = body.charCodeAt(position);
            const nextCode = body.charCodeAt(position + 1);
            switch (code) {
                case 0xfeff:
                case 0x0009:
                case 0x0020:
                    ++position;
                    continue;
                case 0x000a:
                case 0x000d:
                    ++position;
                    if (code === 0x000d && nextCode === 0x000a) {
                        ++position;
                    }
                    ++this.line;
                    this.lineStart = position;
                    continue;
            }
            const token = this.tokenize(position, code);
            if (token) {
                return token;
            }
            throw source.syntaxError(position, code === 0x0027
                ? 'Unexpected single quote character (\'), did you mean to use a double quote (")?'
                : (0, characterClasses_1.isUnicodeScalarValue)(code) || (0, characterClasses_1.isSupplementaryCodePoint)(body, position)
                    ? `Unexpected character: ${source.printCodePointAt(position)}.`
                    : `Invalid character: ${source.printCodePointAt(position)}.`);
        }
        return (0, createToken_1.createToken)(this, TokenKind_1.TokenKind.EOF, bodyLength, bodyLength);
    }
}
exports.Lexer = Lexer;
//# sourceMappingURL=Lexer.js.map