rawsql-ts/dist/esm/parsers/SqlTokenizer.d.ts

High-performance SQL parser and AST analyzer written in TypeScript. Provides fast parsing and advanced transformation capabilities.

import { FormattingLexeme } from '../models/FormattingLexeme';
import { Lexeme } from '../models/Lexeme';
/**
 * Options for tokenization behavior
 */
export interface TokenizeOptions {
    preserveFormatting?: boolean;
}
export interface StatementLexemeResult {
    lexemes: Lexeme[];
    statementStart: number;
    statementEnd: number;
    nextPosition: number;
    rawText: string;
    leadingComments: string[] | null;
}
/**
 * Class responsible for tokenizing SQL input.
 */
export declare class SqlTokenizer {
    /**
     * The input SQL string to be tokenized.
     */
    private input;
    /**
     * Current position within the input string.
     */
    private position;
    /**
     * Manager responsible for handling token readers.
     */
    private readerManager;
    /**
     * Cached start offsets for each line in the input string.
     */
    private lineStartPositions;
    /**
     * Initializes a new instance of the SqlTokenizer.
     */
    constructor(input: string);
    /**
     * Checks if the end of input is reached.
     *
     * @param shift - The shift to consider beyond the current position.
     * @returns True if the end of input is reached; otherwise, false.
     */
    private isEndOfInput;
    /**
     * Checks if more input can be read.
     *
     * @param shift - The shift to consider beyond the current position.
     * @returns True if more input can be read; otherwise, false.
     */
    private canRead;
    private isSelectListModifier;
    private isMeaningfulSelectItemToken;
    /**
     * Tokenizes the input SQL with optional formatting preservation.
     */
    tokenize(): Lexeme[];
    tokenize(options: {
        preserveFormatting: true;
    }): FormattingLexeme[];
    tokenize(options?: TokenizeOptions): Lexeme[] | FormattingLexeme[];
    /**
     * @deprecated Use {@link readLexemes} (correct spelling) instead.
     * This legacy alias remains for backwards compatibility and delegates to the new method.
     */
    readLexmes(): Lexeme[];
    /**
     * Reads the lexemes from the input string.
     *
     * @returns An array of lexemes extracted from the input string.
     * @throws Error if an unexpected character is encountered.
     */
    readLexemes(): Lexeme[];
    /**
     * Tokenizes the input SQL without formatting preservation (internal method)
     */
    private tokenizeBasic;
    readNextStatement(startPosition?: number, carryComments?: string[] | null): StatementLexemeResult | null;
    private hasCommentMetadata;
    private extractLexemes;
    private buildLexemesFromTokenData;
    private skipPastTerminator;
    private mergeComments;
    private relocateOrderByComments;
    private attachCommentsToLexeme;
    /**
     * Skips whitespace characters and SQL comments in the input.
     *
     * @remarks This method updates the position pointer.
     */
    private readComment;
    /**
     * Gets debug information for error reporting.
     *
     * @param errPosition - The position where the error occurred.
     * @returns A string containing the debug position information.
     */
    private getDebugPositionInfo;
    /**
     * Tokenizes the input SQL while preserving formatting information
     */
    private tokenizeWithFormatting;
    private mapToFormattingLexemes;
    /**
     * Find lexeme at a specific position, handling case variations
     */
    private findLexemeAtPosition;
    private isValidLexemeMatch;
    /**
     * Check if character is alphanumeric or underscore (faster than regex)
     */
    private isAlphanumericUnderscore;
    /**
     * Check if character is whitespace (faster than regex)
     */
    private isWhitespace;
    private extractCommentsFromWhitespace;
    /**
     * Skip whitespace and comments from the given position
     */
    private skipWhitespaceAndComments;
    private getLineColumnInfo;
    private getLineColumn;
    private createOrderedLineColumnResolver;
    private ensureLineStartPositions;
}