3h-ast/types/tokenize.d.ts

A simple AST generator.

export declare const tokenizeDefaults: {
    normalSymbols: Set<string>;
    twoCharacterSymbols: Set<string>;
    threeCharacterSymbols: Set<string>;
    fourCharacterSymbols: Set<string>;
    globSymbols: Set<string>;
    numberCharacters: Set<string>;
    extendedNumberCharacters: Set<string>;
    numberSuffixes: Set<string>;
    spaceCharacters: Set<string>;
};
/** dts2md break */
export declare type TokenizeOptions = typeof tokenizeDefaults;
/** dts2md break */
/**
 * Split source code into tokens
 */
export declare const tokenize: (source: string, options?: Partial<{
    normalSymbols: Set<string>;
    twoCharacterSymbols: Set<string>;
    threeCharacterSymbols: Set<string>;
    fourCharacterSymbols: Set<string>;
    globSymbols: Set<string>;
    numberCharacters: Set<string>;
    extendedNumberCharacters: Set<string>;
    numberSuffixes: Set<string>;
    spaceCharacters: Set<string>;
}> | undefined) => string[];