hama-js/dist/node/tokenizer.js

G2P, phoneme-ASR, and P2G inference for Node, Bun, and browsers, powered by a self-contained WASM engine (no onnxruntime).

import { splitTextToJamo } from "./jamo.js";
import vocabData from "./assets/g2p_vocab.json";
export const VOCAB = vocabData;
const encoderTokenToId = new Map(VOCAB.encoder.map((token, idx) => [token, idx]));
const decoderTokenToId = new Map(VOCAB.decoder.map((token, idx) => [token, idx]));
export const encodeText = (text, maxInputLen) => {
    const jamoSeq = splitTextToJamo(text);
    const tokens = jamoSeq.tokens.length ? jamoSeq.tokens : ["<unk>"];
    const indices = jamoSeq.originalIndices.length ? jamoSeq.originalIndices : [-1];
    const ids = tokens.map((token) => encoderTokenToId.get(token) ?? encoderTokenToId.get("<unk>"));
    const length = Math.min(ids.length, maxInputLen);
    const padded = new Array(maxInputLen).fill(BigInt(encoderTokenToId.get("<pad>")));
    for (let i = 0; i < length; i++) {
        padded[i] = BigInt(ids[i]);
    }
    const positionMap = indices.slice(0, length);
    return { ids: padded, length, positionMap: positionMap.length ? positionMap : [-1] };
};
export const decoderIds = {
    pad: decoderTokenToId.get("<pad>"),
    sos: decoderTokenToId.get("<sos>"),
    eos: decoderTokenToId.get("<eos>"),
    unk: decoderTokenToId.get("<unk>"),
};
export const decodeIdsToResult = (ids, attnIndices, positionMap) => {
    const phonemes = [];
    const alignments = [];
    let outOfRangeTokenCount = 0;
    for (let i = 0; i < ids.length; i++) {
        const tokenId = Number(ids[i]);
        if (tokenId === decoderIds.eos)
            break;
        if (tokenId === decoderIds.pad)
            continue;
        if (tokenId === decoderIds.sos && phonemes.length === 0)
            continue;
        const phoneme = VOCAB.decoder[tokenId];
        if (phoneme === undefined) {
            outOfRangeTokenCount += 1;
        }
        const srcPos = Math.max(0, Math.min(Number(attnIndices[i] ?? 0), positionMap.length > 0 ? positionMap.length - 1 : 0));
        const charIndex = positionMap.length > 0 ? positionMap[srcPos] : -1;
        alignments.push({
            phoneme: phoneme ?? VOCAB.decoder[decoderIds.unk],
            phonemeIndex: alignments.length,
            charIndex,
        });
        phonemes.push(phoneme ?? VOCAB.decoder[decoderIds.unk]);
    }
    if (outOfRangeTokenCount > 0 && typeof console !== "undefined") {
        console.warn(`[hama-js] decodeIdsToResult saw ${outOfRangeTokenCount} out-of-range decoder ids; mapped to <unk>.`);
    }
    return { ipa: phonemes.join(""), displayIpa: phonemes.join(""), alignments };
};
const isPunctuation = (ch) => /\p{P}/u.test(ch);
const codePointsWithIndices = (text) => {
    const result = [];
    let offset = 0;
    let charIndex = 0;
    while (offset < text.length) {
        const code = text.codePointAt(offset);
        const ch = String.fromCodePoint(code);
        result.push({ ch, charIndex });
        offset += ch.length;
        charIndex += 1;
    }
    return result;
};
export const prepareTextForPrediction = (text, preserveLiterals) => {
    if (preserveLiterals === "none") {
        return {
            modelText: text,
            charIndexMap: codePointsWithIndices(text).map(({ charIndex }) => charIndex),
        };
    }
    const modelChars = [];
    const charIndexMap = [];
    for (const { ch, charIndex } of codePointsWithIndices(text)) {
        if (isPunctuation(ch))
            continue;
        modelChars.push(ch);
        charIndexMap.push(charIndex);
    }
    return { modelText: modelChars.join(""), charIndexMap };
};
export const buildDisplayIpa = (ipa, alignments, originalText) => {
    const punctuation = codePointsWithIndices(originalText)
        .map(({ ch, charIndex }) => ({ ch, idx: charIndex }))
        .filter(({ ch }) => isPunctuation(ch));
    if (punctuation.length === 0)
        return ipa;
    const parts = [];
    let punctIdx = 0;
    for (const alignment of alignments) {
        while (punctIdx < punctuation.length &&
            punctuation[punctIdx].idx < alignment.charIndex) {
            parts.push(punctuation[punctIdx].ch);
            punctIdx += 1;
        }
        parts.push(alignment.phoneme);
    }
    while (punctIdx < punctuation.length) {
        parts.push(punctuation[punctIdx].ch);
        punctIdx += 1;
    }
    return parts.join("");
};
//# sourceMappingURL=tokenizer.js.map