@palasimi/ipa-cluster/dist/dsl/tokenizer.d.ts

Cluster words with similar IPA transcriptions together

/**
 * Exception thrown by `Tokenizer` when it runs out of tokens.
 */
export declare class EOFError extends Error {
}
/**
 * Represents a type of token.
 */
export declare enum Tag {
    Dot = 0,
    Equals = 1,
    LeftBrace = 2,
    Newline = 3,
    Pipe = 4,
    RightBrace = 5,
    Slash = 6,
    Tilde = 7,
    Underscore = 8,
    Reserved = 9,
    Terminal = 10,
    Variable = 11
}
/**
 * Represents a token in the source program.
 * Contains some information for debugging and error handling.
 */
export type Token = {
    line: number;
    column: number;
    tag: Tag;
    literal: string;
};
/**
 * Tokenizes code.
 * Returns an array of `Token`s.
 */
export declare function tokenize(code: string): Token[];
//# sourceMappingURL=tokenizer.d.ts.map