@adguard/agtree/dist/types/tokenizer/tokenizer.d.ts

Tool set for working with adblock filter lists

/**
 * Token types.
 */
export declare const enum TokenType {
    /**
     * End of file (end of input).
     */
    Eof = 0,
    /**
     * Whitespace.
     */
    Whitespace = 1,
    /**
     * Line break (`\r\n` or just `\n`)
     */
    LineBreak = 2,
    /**
     * Escaped character, e.g. `\'`, `\"`, `\\`, etc.
     */
    Escaped = 3,
    /**
     * Identifier.
     * Any character sequence that contains letters, numbers, hyphens, and underscores.
     */
    Ident = 4,
    /**
     * Cosmetic rule separator, e.g. `##`.
     */
    CosmeticSeparator = 5,
    /**
     * Allowlist cosmetic rule separator, e.g. `#@#`.
     */
    AllowlistCosmeticSeparator = 6,
    /**
     * Raw content after cosmetic rule separator.
     * For example, no need to tokenize CSS with this tokenizer after the `##`, `#?#`, etc. separators,
     * so we use this token type as an optimization strategy.
     */
    RawContent = 7,
    /**
     * Equals: `=`.
     */
    EqualsSign = 8,
    /**
     * Slash: `/`.
     */
    Slash = 9,
    /**
     * Dollar: `$`.
     */
    DollarSign = 10,
    /**
     * Comma: `,`.
     */
    Comma = 11,
    /**
     * Open parenthesis: `(`.
     */
    OpenParen = 12,
    /**
     * Close parenthesis: `)`.
     */
    CloseParen = 13,
    /**
     * Open brace: `{`.
     */
    OpenBrace = 14,
    /**
     * Close brace: `}`.
     */
    CloseBrace = 15,
    /**
     * Open square: `[`.
     */
    OpenSquare = 16,
    /**
     * Close square: `]`.
     */
    CloseSquare = 17,
    /**
     * Pipe: `|`.
     */
    Pipe = 18,
    /**
     * At: `@`.
     */
    AtSign = 19,
    /**
     * Asterisk: `*`.
     */
    Asterisk = 20,
    /**
     * Quote: `"`.
     */
    Quote = 21,
    /**
     * Apostrophe: `'`.
     */
    Apostrophe = 22,
    /**
     * Exclamation: `!`.
     */
    ExclamationMark = 23,
    /**
     * Hashmark: `#`.
     */
    HashMark = 24,
    /**
     * Plus: `+`.
     */
    PlusSign = 25,
    /**
     * And: `&`.
     */
    AndSign = 26,
    /**
     * Tilde: `~`.
     */
    Tilde = 27,
    /**
     * Caret: `^`.
     */
    Caret = 28,
    /**
     * Dot: `.`.
     */
    Dot = 29,
    /**
     * Semicolon: `;`.
     */
    Semicolon = 30,
    /**
     * Any other character.
     */
    Symbol = 31
}
/**
 * Function to stop tokenization.
 * Does not have effect on EOF.
 */
type StopFunction = () => void;
/**
 * Function to skip the current rule.
 * Does not have effect on EOF.
 */
type SkipFunction = () => void;
/**
 * Function to jump to a specific relative position.
 * If the position is out of bounds, it will be clamped.
 * Does not have effect on EOF.
 *
 * @param count The number of positions to jump.
 */
type JumpFunction = (count: number) => void;
/**
 * Callback function for tokenization.
 *
 * @param token The token type.
 * @param start The start position of the token.
 * @param end The end position of the token.
 * @param stop The function to stop tokenization.
 * @param skip The function to skip the current rule.
 * @param jump The function to jump to a specific relative position.
 */
export type OnTokenCallback = (token: TokenType, start: number, end: number, stop: StopFunction, skip: SkipFunction, jump: JumpFunction) => void;
/**
 * Tokenizes a string.
 *
 * @param string The string to tokenize.
 * @param onToken The callback function to handle tokens.
 */
export declare const tokenize: (string: string, onToken: OnTokenCallback) => void;
export {};