phonemize/dist/tokenizer.d.ts

Fast phonemizer with rule-based G2P prediction. Pure JavaScript implementation.

/**
 * Text tokenization and phoneme processing system
 * Handles language detection, preprocessing, and format conversion
 */
import { LanguageRegistry } from "./g2p";
/**
 * Configuration options for tokenizer behavior
 */
export interface TokenizerOptions {
    /** Remove stress markers from output */
    stripStress?: boolean;
    /**
     * Output format (IPA, ARPABET, or Zhuyin)
     *
     * Note: Non-chinese in zhuyin format will be converted to IPA
     **/
    format?: "ipa" | "arpabet" | "zhuyin";
    /** Token separator in output string */
    separator?: string;
    /** Convert non-Latin text to ASCII approximation */
    anyAscii?: boolean;
    /** Chinese tone format: 'unicode' (˧˩˧) or 'arrow' (↓↗↘→). Only applies when format is 'ipa' */
    toneFormat?: "unicode" | "arrow";
    /**
     * How to render the affricates /dʒ/ and /tʃ/ in IPA output:
     *   'separate' (default) — two characters dʒ / tʃ (current IPA standard)
     *   'ligature'           — single glyphs ʤ / ʧ (withdrawn from IPA 1989,
     *                          but still consumed by some TTS systems)
     *   'tie-bar'            — d͡ʒ / t͡ʃ with the U+0361 tie bar (most precise)
     * Only applies when format is 'ipa'.
     */
    affricates?: "separate" | "ligature" | "tie-bar";
    /**
     * Preferred language tag (BCP 47, e.g. "en", "en-GB", "zh").
     *
     * When set, the tokenizer skips Unicode-based auto-detection for words
     * that don't clearly belong to a different script and routes them to
     * a G2P processor matching this tag. Words written in a script that
     * unambiguously identifies another language (e.g. Han for Chinese)
     * still go through their script-detected processor.
     */
    language?: string;
    /**
     * G2P registry to use for phoneme prediction. Defaults to the global
     * registry populated by the package's `useProcessor()` calls. Pass a custom
     * registry (or use `createPhonemizer()`) for isolated multi-instance
     * setups.
     */
    registry?: LanguageRegistry;
}
/**
 * Individual phoneme token with metadata
 */
export interface PhonemeToken {
    /** IPA or ARPABET phoneme string */
    phoneme: string;
    /** Original word/text */
    word: string;
    /** Position in original text */
    position: number;
}
/**
 * Language segment for multilingual processing
 */
interface LanguageSegment {
    text: string;
    language: string;
    startIndex: number;
}
/**
 * Preprocessing result with language information
 */
interface PreprocessResult {
    text: string;
    languageMap: Record<string, string>;
    segments: LanguageSegment[];
    /** Document-wide dominant language (from `analyzeText`). */
    primary?: string;
    /** Whether Han chars in this text should be routed as Japanese. */
    hanIsJa: boolean;
}
/**
 * Main tokenizer class for phoneme processing
 */
export declare class Tokenizer {
    protected readonly options: Required<Omit<TokenizerOptions, "language" | "registry">> & {
        language: string | undefined;
    };
    protected readonly registry: LanguageRegistry;
    constructor(options?: TokenizerOptions);
    /**
     * Preprocess text: per-language normalization (numbers, abbreviations, …)
     * via each script-run's `LanguageProcessor.preProcess`, then optional
     * anyAscii Latinization.
     *
     * The document-level analysis (`primary`, `hanIsJa`) is computed on the
     * **original** text — preProcess for some languages (notably Japanese)
     * Latinizes its segments to give the downstream G2P a clean Hepburn
     * romaji stream, which would otherwise erase the script signal and
     * cause the tokenizer to re-route everything as English. Running
     * analyzeText first preserves the language identity for the per-token
     * dispatch's primary-language fallback.
     *
     * Expansion happens before anyAscii so non-Latin expanders (e.g. Russian)
     * operate on their original script — anyAscii would otherwise Latinize
     * the run and strip the language signal.
     */
    protected _preprocess(text: string): PreprocessResult;
    /**
     * Detect languages for words and create character-level segments.
     *
     * `hanIsJa` (from the document-level `analyzeText`) makes both the
     * per-character segment language and the per-word `languageMap` agree
     * with whichever G2P will actually be dispatched to.
     */
    private _detectLanguagesAndSegment;
    /**
     * Apply anyAscii conversion while preserving Chinese text
     */
    private _applyAnyAscii;
    /**
     * Fast character-level language detection
     */
    private _detectCharLanguage;
    /**
     * Post-process phonemes for format conversion and cleanup
     */
    protected _postProcess(phonemes: string): string;
    private _predict;
    /**
     * Core token processing method that handles both simple and detailed tokenization
     */
    private _processTokens;
    /**
     * Core tokenization method - converts text to phoneme array
     */
    tokenize(text: string): string[];
    /**
     * Smart tokenization using efficient regex patterns
     */
    private _smartTokenize;
    /**
     * Convert text to phoneme string with specified separator
     */
    tokenizeToString(text: string): string;
    /**
     * Convert text to detailed phoneme tokens with metadata
     */
    tokenizeToTokens(text: string): PhonemeToken[];
}
export {};