@unisnips/ultisnips/lib/parse/tokenizer.js

Utilities for converting ultisnips in unisnips project

"use strict";
/**
 * Convert snippet text file into logical units called Tokens.
 * Ports https://github.com/SirVer/ultisnips/blob/master/pythonx/UltiSnips/snippet/parsing/lexer.py
 */
var __extends = (this && this.__extends) || (function () {
    var extendStatics = function (d, b) {
        extendStatics = Object.setPrototypeOf ||
            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
            function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
        return extendStatics(d, b);
    };
    return function (d, b) {
        extendStatics(d, b);
        function __() { this.constructor = d; }
        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
    };
})();
var __values = (this && this.__values) || function(o) {
    var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
    if (m) return m.call(o);
    if (o && typeof o.length === "number") return {
        next: function () {
            if (o && i >= o.length) o = void 0;
            return { value: o && o[i++], done: !o };
        }
    };
    throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
var __read = (this && this.__read) || function (o, n) {
    var m = typeof Symbol === "function" && o[Symbol.iterator];
    if (!m) return o;
    var i = m.call(o), r, ar = [], e;
    try {
        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
    }
    catch (error) { e = { error: error }; }
    finally {
        try {
            if (r && !r.done && (m = i["return"])) m.call(i);
        }
        finally { if (e) throw e.error; }
    }
    return ar;
};
Object.defineProperty(exports, "__esModule", { value: true });
/* eslint-disable @typescript-eslint/no-use-before-define */
var position_1 = require("../util/position");
var control_flow_1 = require("@bestminr/control-flow");
var StopIteration = /** @class */ (function (_super) {
    __extends(StopIteration, _super);
    function StopIteration() {
        return _super !== null && _super.apply(this, arguments) || this;
    }
    return StopIteration;
}(control_flow_1.ExtensibleError));
exports.StopIteration = StopIteration;
/**
 * Helper class to make iterating over text easier.
 * A simplified char stream
 */
var TextIterator = /** @class */ (function () {
    function TextIterator(text, startPosition) {
        this.idx = 0;
        this.text = text;
        this.textLineContents = text.split('\n');
        this.line = startPosition.line;
        this.col = startPosition.column;
        this.startPosition = startPosition;
    }
    TextIterator.prototype.peek = function (len) {
        if (len === void 0) { len = 1; }
        if (len > 1)
            return this.text.substr(this.idx, len);
        else
            return this.text[this.idx];
    };
    /** return the next n characters */
    TextIterator.prototype.next = function (n) {
        if (n === void 0) { n = 1; }
        if (n === 1)
            return this.nextOne();
        var chars = [];
        for (var i = 0; i < n; i++) {
            chars.push(this.nextOne());
        }
        return chars.join('');
    };
    /** roll back the stream n characters, won't return chars */
    TextIterator.prototype.rollback = function (n) {
        if (n === void 0) { n = 1; }
        if (n === 1)
            return this.rollbackOne();
        for (var i = 0; i < n; i++) {
            this.rollbackOne();
        }
    };
    /** return the next character */
    TextIterator.prototype.nextOne = function () {
        if (this.idx > this.text.length) {
            throw new StopIteration();
        }
        var rv = this.text[this.idx];
        if (this.text[this.idx] === '\n') {
            this.line += 1;
            this.col = 0;
        }
        else {
            this.col += 1;
        }
        this.idx += 1;
        return rv;
    };
    TextIterator.prototype.rollbackOne = function () {
        if (this.idx < 0) {
            return;
        }
        if (this.col <= 0) {
            this.line -= 1;
            this.col = this.textLineContents[this.line].length - 1;
        }
        else {
            this.col -= 1;
        }
        this.idx -= 1;
    };
    Object.defineProperty(TextIterator.prototype, "position", {
        get: function () {
            var startOffset = this.startPosition.offset || 0;
            return new position_1.TextPosition(this.line, this.col, startOffset + this.idx);
        },
        enumerable: true,
        configurable: true
    });
    return TextIterator;
}());
exports.TextIterator = TextIterator;
var CHARS = {
    digits: '0123456789'.split(''),
};
function parseIndexNumber(iter) {
    var rv = '';
    var peekedV = iter.peek();
    while (peekedV && CHARS.digits.includes(peekedV)) {
        rv += iter.next();
        peekedV = iter.peek();
    }
    return parseInt(rv);
}
function parseTillClosingBrace(iter) {
    var braceDepth = 1;
    var retChars = [];
    while (true) {
        if (EscapeCharToken.startsWithChar(iter, '{}')) {
            retChars.push(iter.next(2));
        }
        else {
            var char = iter.next();
            if (char === '{')
                braceDepth += 1;
            else if (char === '}')
                braceDepth -= 1;
            if (braceDepth < 1) {
                break;
            }
            retChars.push(char);
        }
    }
    return retChars.join('');
}
exports.parseTillClosingBrace = parseTillClosingBrace;
/**
 * Returns all chars till a non-escaped char is found.
 *
 * Will also consume the closing char, but and return it as second return value
 */
function parseTillUnescapedChar(iter, str, validWhenEndOfText) {
    var e_1, _a;
    if (validWhenEndOfText === void 0) { validWhenEndOfText = false; }
    var retChars = [];
    var char;
    try {
        while (true) {
            var chars = str.split('');
            var escaped = false;
            try {
                for (var chars_1 = (e_1 = void 0, __values(chars)), chars_1_1 = chars_1.next(); !chars_1_1.done; chars_1_1 = chars_1.next()) {
                    var c = chars_1_1.value;
                    char = c;
                    if (EscapeCharToken.startsWithChar(iter, char)) {
                        retChars.push(iter.next(2));
                        escaped = true;
                    }
                }
            }
            catch (e_1_1) { e_1 = { error: e_1_1 }; }
            finally {
                try {
                    if (chars_1_1 && !chars_1_1.done && (_a = chars_1.return)) _a.call(chars_1);
                }
                finally { if (e_1) throw e_1.error; }
            }
            if (!escaped) {
                char = iter.next();
                if (chars.includes(char)) {
                    break;
                }
                else {
                }
                retChars.push(char);
            }
        }
    }
    catch (error) {
        if (error instanceof StopIteration) {
            if (!validWhenEndOfText) {
                throw error;
            }
        }
        else {
            throw error;
        }
    }
    return [retChars.join(''), char];
}
exports.parseTillUnescapedChar = parseTillUnescapedChar;
/**
 * Represents a Token as parsed from a snippet definition.
 */
var Token = /** @class */ (function () {
    function Token(iter, opts) {
        if (opts === void 0) { opts = {}; }
        this.initialText = '';
        this.isTransformable = false;
        this.parent = opts.parent;
        this.start = iter.position;
        this.parse(iter, opts.indent);
        this.end = iter.position;
    }
    // transform?: TransformationToken
    Token.startsHere = function (iter) {
        var args = [];
        for (var _i = 1; _i < arguments.length; _i++) {
            args[_i - 1] = arguments[_i];
        }
        return false;
    };
    Object.defineProperty(Token.prototype, "tokenType", {
        /**
         * will be serialized to TokenNode
         */
        get: function () {
            return 'Token';
        },
        enumerable: true,
        configurable: true
    });
    Token.prototype.getTokenNodeData = function () {
        return {};
    };
    Token.prototype.toTokenNode = function () {
        return {
            type: this.tokenType,
            position: {
                start: this.start.toUnistPosition(),
                end: this.end.toUnistPosition(),
            },
            data: this.getTokenNodeData(),
        };
    };
    return Token;
}());
exports.Token = Token;
var TabStopToken = /** @class */ (function (_super) {
    __extends(TabStopToken, _super);
    function TabStopToken() {
        var _this = _super !== null && _super.apply(this, arguments) || this;
        _this.isTransformable = true;
        return _this;
    }
    TabStopToken.startsHere = function (iter) {
        return this.PATTERN.test(iter.peek(10));
    };
    TabStopToken.prototype.getTokenNodeData = function () {
        return { number: this.number };
    };
    Object.defineProperty(TabStopToken.prototype, "tokenType", {
        get: function () {
            return 'TabStop';
        },
        enumerable: true,
        configurable: true
    });
    TabStopToken.prototype.parse = function (iter) {
        iter.next(); // $
        iter.next(); // {
        this.number = parseIndexNumber(iter);
        if (iter.peek() === ':') {
            this.hasColon = true;
            iter.next();
        }
        this.initialText = parseTillClosingBrace(iter);
    };
    TabStopToken.PATTERN = /^\$\{\d+[:}]?/;
    return TabStopToken;
}(Token));
exports.TabStopToken = TabStopToken;
var VisualToken = /** @class */ (function (_super) {
    __extends(VisualToken, _super);
    function VisualToken() {
        var _this = _super !== null && _super.apply(this, arguments) || this;
        _this.isTransformable = true;
        _this.alternativeText = '';
        _this.options = null;
        return _this;
    }
    VisualToken.startsHere = function (iter) {
        return this.PATTERN.test(iter.peek(10));
    };
    Object.defineProperty(VisualToken.prototype, "tokenType", {
        get: function () {
            return 'Visual';
        },
        enumerable: true,
        configurable: true
    });
    VisualToken.prototype.toString = function () {
        return "VisualToken(" + this.start + "," + this.end + ")";
    };
    VisualToken.prototype.parse = function (iter) {
        iter.next(8); // ${VISUAL
        if (iter.peek() === ':') {
            iter.next();
        }
        var _a = __read(parseTillUnescapedChar(iter, '/}'), 2), text = _a[0], closingChar = _a[1];
        this.alternativeText = unescape(text);
        if (closingChar === '/') {
            try {
                this.search = parseTillUnescapedChar(iter, '/')[0];
                this.replace = parseTillUnescapedChar(iter, '/')[0];
                this.options = parseTillClosingBrace(iter);
            }
            catch (error) {
                throw new Error("Invalid ${VISUAL} transformation! Forgot to escape a '/'?");
            }
        }
        else {
            this.search = null;
            this.replace = null;
        }
    };
    // ${VISUAL}
    VisualToken.PATTERN = /^\$\{VISUAL[:}\/]/;
    return VisualToken;
}(Token));
exports.VisualToken = VisualToken;
// export class TransformationToken extends Token {
//   search: string
//   replace: string
//   options: any = null
//   static startsHere(iter: TextIterator) {
//     return iter.peek() === '/'
//   }
//   get tokenType() {
//     return 'Transformation'
//   }
//   getTokenNodeData() {
//     return pick(this, [
//       'number',
//       'search',
//       'replace',
//       'options',
//     ] as Array<keyof this>);
//   }
//   protected parse(iter: TextIterator) {
//     iter.next() // /
//     this.search = parseTillUnescapedChar(iter, '/')[0]
//     this.replace = parseTillUnescapedChar(iter, '/')[0]
//     this.options = parseTillClosingBrace(iter)
//   }
// }
var MirrorToken = /** @class */ (function (_super) {
    __extends(MirrorToken, _super);
    function MirrorToken() {
        return _super !== null && _super.apply(this, arguments) || this;
    }
    MirrorToken.startsHere = function (iter) {
        return this.PATTERN.test(iter.peek(10));
    };
    Object.defineProperty(MirrorToken.prototype, "tokenType", {
        get: function () {
            return 'Mirror';
        },
        enumerable: true,
        configurable: true
    });
    MirrorToken.prototype.toString = function () {
        return "MirrorToken(" + this.start.toString() + "," + this.end.toString() + "," + this.number + ")";
    };
    MirrorToken.prototype.getTokenNodeData = function () {
        return { number: this.number };
    };
    MirrorToken.prototype.parse = function (iter) {
        iter.next(); // $
        this.number = parseIndexNumber(iter);
    };
    MirrorToken.PATTERN = /^\$\d+/;
    return MirrorToken;
}(Token));
exports.MirrorToken = MirrorToken;
/**
 * Represents UniSnips' builtin variable
 */
var UniSnipsVariableToken = /** @class */ (function (_super) {
    __extends(UniSnipsVariableToken, _super);
    function UniSnipsVariableToken() {
        return _super !== null && _super.apply(this, arguments) || this;
    }
    // static VALID_NAME_PATTERN = /[0-9A-Z_]/
    UniSnipsVariableToken.startsHere = function (iter) {
        // $UNI_*
        return iter.peek(5) === '$UNI_';
    };
    UniSnipsVariableToken.prototype.toString = function () {
        return "UniSnipsVariableToken(" + this.start.toString() + "," + this.end.toString() + "," + this.name + ")";
    };
    UniSnipsVariableToken.prototype.parse = function (iter) {
        iter.next(5); // $UNI_
        var _a = __read(parseTillUnescapedChar(iter, '$,\n./ ', true), 1), restName = _a[0];
        this.name = 'UNI_' + restName;
    };
    return UniSnipsVariableToken;
}(Token));
exports.UniSnipsVariableToken = UniSnipsVariableToken;
var EscapeCharToken = /** @class */ (function (_super) {
    __extends(EscapeCharToken, _super);
    function EscapeCharToken() {
        return _super !== null && _super.apply(this, arguments) || this;
    }
    EscapeCharToken.startsHere = function (iter, charsOrIndent) {
        if (typeof charsOrIndent === 'string') {
            return this.startsWithChar(iter, charsOrIndent);
        }
    };
    EscapeCharToken.startsWithChar = function (iter, validEscapeChars) {
        if (validEscapeChars === void 0) { validEscapeChars = '{}\\$`'; }
        // console.log('cha', validEscapeChars)
        var chars = iter.peek(2);
        return chars.length === 2 && chars[0] === '\\' && validEscapeChars.indexOf(chars[1]) > -1;
    };
    Object.defineProperty(EscapeCharToken.prototype, "tokenType", {
        get: function () {
            return 'EscapeChar';
        },
        enumerable: true,
        configurable: true
    });
    EscapeCharToken.prototype.parse = function (iter) {
        iter.next(); // \
        this.initialText = iter.next();
    };
    return EscapeCharToken;
}(Token));
exports.EscapeCharToken = EscapeCharToken;
/**
 * There are various script types, but they are all wrapped by '`'
 *
 * - `ehco "unisnips"` , shell
 * - `!js console.log('hello')`, js
 * - `!p snip.rv = "hi"`, python
 */
var ScriptCodeToken = /** @class */ (function (_super) {
    __extends(ScriptCodeToken, _super);
    function ScriptCodeToken() {
        return _super !== null && _super.apply(this, arguments) || this;
    }
    ScriptCodeToken.startsHere = function (iter) {
        return iter.peek() === '`';
    };
    Object.defineProperty(ScriptCodeToken.prototype, "tokenType", {
        get: function () {
            return 'ScriptCode';
        },
        enumerable: true,
        configurable: true
    });
    ScriptCodeToken.prototype.getTokenNodeData = function () {
        return {
            scriptType: this.scriptType,
            scriptCode: this.scriptCode,
        };
    };
    ScriptCodeToken.prototype.parse = function (iter) {
        iter.next(); // `
        var nextChars = iter.peek(5);
        var scriptType;
        if (this.containsWord(nextChars, '!')) {
            scriptType = 'shell';
            iter.next(2);
        }
        else if (this.containsWord(nextChars, '!p')) {
            scriptType = 'python';
            iter.next(3);
        }
        else if (this.containsWord(nextChars, '!v')) {
            scriptType = 'vim';
            iter.next(3);
        }
        else if (this.containsWord(nextChars, '!js')) {
            scriptType = 'js';
            iter.next(4);
        }
        if (scriptType) {
            this.scriptType = scriptType;
            var _a = __read(parseTillUnescapedChar(iter, '`'), 1), content = _a[0];
            this.scriptCode = content;
        }
    };
    ScriptCodeToken.prototype.containsWord = function (chars, word, validSeperator) {
        if (validSeperator === void 0) { validSeperator = [' ', '\n']; }
        var charsToCheck = chars.substr(0, word.length + 1);
        var lastChar = charsToCheck[charsToCheck.length - 1];
        return charsToCheck.startsWith(word) && validSeperator.includes(lastChar);
    };
    return ScriptCodeToken;
}(Token));
exports.ScriptCodeToken = ScriptCodeToken;
var EndOfTextToken = /** @class */ (function (_super) {
    __extends(EndOfTextToken, _super);
    function EndOfTextToken() {
        return _super !== null && _super.apply(this, arguments) || this;
    }
    Object.defineProperty(EndOfTextToken.prototype, "tokenType", {
        get: function () {
            return 'EndOfText';
        },
        enumerable: true,
        configurable: true
    });
    EndOfTextToken.prototype.parse = function () {
        // nothing
    };
    return EndOfTextToken;
}(Token));
exports.EndOfTextToken = EndOfTextToken;
function tokenize(text, indent, position, allowedTokens) {
    var e_2, _a;
    var iter = new TextIterator(text, position);
    var tokens = [];
    try {
        while (true) {
            var doneSomething = false;
            try {
                for (var allowedTokens_1 = (e_2 = void 0, __values(allowedTokens)), allowedTokens_1_1 = allowedTokens_1.next(); !allowedTokens_1_1.done; allowedTokens_1_1 = allowedTokens_1.next()) {
                    var tokenCtor = allowedTokens_1_1.value;
                    if (tokenCtor.startsHere(iter, indent)) {
                        var token = new tokenCtor(iter, { indent: indent });
                        tokens.push(token);
                        doneSomething = true;
                        // if (token.isTransformable && token.transform) {
                        //   tokens.push(token.transform)
                        // }
                        break;
                    }
                    else {
                    }
                }
            }
            catch (e_2_1) { e_2 = { error: e_2_1 }; }
            finally {
                try {
                    if (allowedTokens_1_1 && !allowedTokens_1_1.done && (_a = allowedTokens_1.return)) _a.call(allowedTokens_1);
                }
                finally { if (e_2) throw e_2.error; }
            }
            if (!doneSomething) {
                iter.next();
            }
        }
    }
    catch (error) {
        if (error instanceof StopIteration) {
            return tokens;
        }
        throw error;
    }
    return tokens;
}
exports.tokenize = tokenize;
//# sourceMappingURL=tokenizer.js.map