@stacksjs/stx/dist/parser/tokenizer.d.ts

A performant UI Framework. Powered by Bun.

/**
 * Find matching closing delimiter, respecting nesting and strings
 */
export declare function findMatchingDelimiter(source: string, openChar: string, closeChar: string, startPos?: number): number;
/**
 * Extract a balanced expression from source starting at position
 */
export declare function extractBalancedExpression(source: string, startPos: number): {
  expression: string
  endPos: number
};
/**
 * Split expression by pipe (|) for filters, respecting nesting
 */
export declare function splitByPipe(expression: string): string[];
export declare interface Token {
  type: TokenType
  value: string
  start: number
  end: number
  line: number
  column: number
}
export declare interface TokenizerState {
  pos: number
  line: number
  column: number
}
/**
 * Tokenizer for Template Expressions
 *
 * A character-by-character tokenizer that properly handles:
 * - Nested braces, brackets, and parentheses
 * - String literals (single, double, and template)
 * - Escaped characters within strings
 * - Template literal expressions `${...}`
 *
 * @module parser/tokenizer
 */
// =============================================================================
// Token Types
// =============================================================================
export type TokenType = | 'STRING' // 'string' or "string"
  | 'TEMPLATE_STRING' // `template ${expr} string`
  | 'NUMBER'
  | 'IDENTIFIER'
  | 'OPERATOR'
  | 'PUNCTUATION' // ( ) [ ] { } , ; :
  | 'WHITESPACE'
  | 'COMMENT'
  | 'EXPRESSION_START' // {{
  | 'EXPRESSION_END' // }}
  | 'RAW_START' // {!!
  | 'RAW_END' // !!}
  | 'DIRECTIVE' // @name
  | 'EOF'
/**
 * Tokenizer for template expressions
 */
export declare class Tokenizer {
  private source: string;
  private pos: number;
  private line: number;
  private column: number;
  private tokens: Token[];
  constructor(source: string);
  tokenize(): Token[];
  private nextToken(): Token | null;
  private readWhitespace(start: number, startLine: number, startColumn: number): Token;
  private readLineComment(start: number, startLine: number, startColumn: number): Token;
  private readBlockComment(start: number, startLine: number, startColumn: number): Token;
  private readString(quote: string, start: number, startLine: number, startColumn: number): Token;
  private readTemplateString(start: number, startLine: number, startColumn: number): Token;
  private readNumber(start: number, startLine: number, startColumn: number): Token;
  private readIdentifier(start: number, startLine: number, startColumn: number): Token;
  private readDirective(start: number, startLine: number, startColumn: number): Token;
  private advance(count?: number): void;
  private peek(offset: number): string | undefined;
  saveState(): TokenizerState;
  restoreState(state: TokenizerState): void;
}