@unisnips/ultisnips/lib/parse/tokenizer.d.ts

Utilities for converting ultisnips in unisnips project

/**
 * Convert snippet text file into logical units called Tokens.
 * Ports https://github.com/SirVer/ultisnips/blob/master/pythonx/UltiSnips/snippet/parsing/lexer.py
 */
import { TextPosition } from '../util/position';
import { ExtensibleError } from '@bestminr/control-flow';
import { SnippetPlaceholder, TokenNode } from '@unisnips/core';
export declare class StopIteration extends ExtensibleError {
}
/**
 * Helper class to make iterating over text easier.
 * A simplified char stream
 */
export declare class TextIterator {
    protected text: string;
    protected line: number;
    protected col: number;
    protected textLineContents: string[];
    protected idx: number;
    /**
     * holds startPosition for parsing inside TabStop,
     * but will still start idx from 0
     */
    protected startPosition: TextPosition;
    constructor(text: string, startPosition: TextPosition);
    peek(len?: number): string;
    /** return the next n characters */
    next(n?: number): string;
    /** roll back the stream n characters, won't return chars */
    rollback(n?: number): void;
    /** return the next character */
    protected nextOne(): string;
    protected rollbackOne(): void;
    get position(): TextPosition;
}
export declare function parseTillClosingBrace(iter: TextIterator): string;
/**
 * Returns all chars till a non-escaped char is found.
 *
 * Will also consume the closing char, but and return it as second return value
 */
export declare function parseTillUnescapedChar(iter: TextIterator, str: string, validWhenEndOfText?: boolean): string[];
declare type TokenStatics = {
    startsHere(iter: TextIterator, ...args: any[]): boolean;
};
declare type TokenInitOpts = {
    indent?: TextPosition;
    parent?: Token;
};
/**
 * Represents a Token as parsed from a snippet definition.
 */
export declare abstract class Token {
    initialText: string;
    start: TextPosition;
    end: TextPosition;
    parent?: Token;
    isTransformable: boolean;
    static startsHere(iter: TextIterator, ...args: any[]): boolean;
    constructor(iter: TextIterator, opts?: TokenInitOpts);
    protected abstract parse(iter: TextIterator, indent: TextPosition): void;
    /**
     * will be serialized to TokenNode
     */
    get tokenType(): string;
    getTokenNodeData(): {};
    toTokenNode(): TokenNode<any>;
}
export declare class TabStopToken extends Token {
    static PATTERN: RegExp;
    isTransformable: boolean;
    number: number;
    /** if tabstop has ':', further marker construct will skip transformer */
    hasColon: boolean;
    static startsHere(iter: TextIterator): boolean;
    getTokenNodeData(): {
        number: number;
    };
    get tokenType(): string;
    protected parse(iter: TextIterator): void;
}
export declare class VisualToken extends Token {
    static PATTERN: RegExp;
    static startsHere(iter: TextIterator): boolean;
    isTransformable: boolean;
    alternativeText: string;
    search: string;
    replace: string;
    options: any;
    get tokenType(): string;
    toString(): string;
    protected parse(iter: TextIterator): void;
}
export declare class MirrorToken extends Token {
    static PATTERN: RegExp;
    static startsHere(iter: TextIterator): boolean;
    number: number;
    get tokenType(): string;
    toString(): string;
    getTokenNodeData(): {
        number: number;
    };
    protected parse(iter: TextIterator): void;
}
/**
 * Represents UniSnips' builtin variable
 */
export declare class UniSnipsVariableToken extends Token {
    static startsHere(iter: TextIterator): boolean;
    name: string;
    toString(): string;
    protected parse(iter: TextIterator): void;
}
export declare class EscapeCharToken extends Token {
    static startsHere(iter: TextIterator, charsOrIndent: any): boolean;
    static startsWithChar(iter: TextIterator, validEscapeChars?: string): boolean;
    get tokenType(): string;
    protected parse(iter: TextIterator): void;
}
declare type ScriptType = SnippetPlaceholder['scriptInfo']['scriptType'];
/**
 * There are various script types, but they are all wrapped by '`'
 *
 * - `ehco "unisnips"` , shell
 * - `!js console.log('hello')`, js
 * - `!p snip.rv = "hi"`, python
 */
export declare class ScriptCodeToken extends Token {
    static startsHere(iter: TextIterator): boolean;
    scriptType: ScriptType;
    scriptCode: string;
    get tokenType(): string;
    getTokenNodeData(): {
        scriptType: "python" | "shell" | "vim" | "js";
        scriptCode: string;
    };
    protected parse(iter: TextIterator): void;
    protected containsWord(chars: string, word: string, validSeperator?: string[]): boolean;
}
export declare class EndOfTextToken extends Token {
    get tokenType(): string;
    protected parse(): void;
}
export declare type TokenCtor = new (iter: TextIterator, opts: TokenInitOpts) => Token;
export declare type TokenClass = TokenCtor & TokenStatics;
export declare function tokenize(text: string, indent: TextPosition, position: TextPosition, allowedTokens: TokenClass[]): Token[];
export {};