hama-js
Version:
G2P, phoneme-ASR, and P2G inference for Node, Bun, and browsers, powered by a self-contained WASM engine (no onnxruntime).
1,385 lines • 57.7 kB
JavaScript
import { splitTextToJamo } from "./jamo.js";
const DEFAULT_SCAN_OPTIONS = {
language: "en",
spanUnit: "character",
maxDistanceRatio: 0.2,
minDistance: 0,
maxDistance: null,
thresholdBasis: "phonemes",
wordBoundaryMode: "flexible",
tokenSlack: 1,
qgramSize: 2,
maxTermPronunciations: 4,
verifier: "auto",
scoring: "hybrid",
phonemeWeight: 0.85,
textWeight: 0.15,
minScore: 0,
resolveOverlaps: "best_non_overlapping",
allowShortFuzzy: false,
returnPhonemes: false,
debug: false,
};
const DEFAULT_REPLACE_OPTIONS = {
language: "en",
spanUnit: "character",
maxDistanceRatio: 0.2,
minDistance: 0,
maxDistance: null,
thresholdBasis: "phonemes",
wordBoundaryMode: "flexible",
tokenSlack: 1,
qgramSize: 2,
maxTermPronunciations: 4,
verifier: "auto",
scoring: "hybrid",
phonemeWeight: 0.85,
textWeight: 0.15,
minScore: 0.72,
allowShortFuzzy: false,
returnPhonemes: false,
debug: false,
replacementSource: "canonical",
caseStrategy: "canonical",
conflictPolicy: "weighted_interval",
ambiguousPolicy: "skip",
ambiguityMargin: 0.05,
includeUnchanged: false,
includeDiscarded: true,
keepScanMatches: false,
};
const APOSTROPHE_CHARS = new Set(["'", "\u2019", "\u2018", "\u02bc", "`", "\u00b4", "\uff07"]);
const DASH_CHARS = new Set(["-", "\u2010", "\u2011", "\u2012", "\u2013", "\u2014", "\u2015", "\u2212", "\ufe63", "\uff0d"]);
const WORD_JOINERS = new Set([...APOSTROPHE_CHARS, ...DASH_CHARS]);
export async function pronunciationScanWithModel(model, text, terms, options = {}) {
const merged = mergeScanOptions(options);
if (!text || terms.length === 0) {
return { matches: [], stats: emptyScanStats(0) };
}
const phoneEncoder = new PhoneEncoder();
const qgramEncoder = new QGramEncoder();
const tokens = await prepareTokens(text, model, merged, phoneEncoder);
const maxInputLen = resolvePredictorMaxInputLen(model);
const compiled = await compileVariants(terms, model, merged, phoneEncoder, qgramEncoder, maxInputLen);
const baseStats = emptyScanStats(tokens.length);
baseStats.rejectedByInputLimit = compiled.rejectedByInputLimit;
if (compiled.variants.length === 0) {
return { matches: [], stats: baseStats };
}
const { matches, stats } = merged.spanUnit === "character"
? await scanCompiledByCharacters(text, tokens, compiled, merged, qgramEncoder, model, phoneEncoder, maxInputLen)
: scanCompiled(text, tokens, compiled, merged, qgramEncoder);
stats.rejectedByInputLimit = (stats.rejectedByInputLimit ?? 0) + compiled.rejectedByInputLimit;
const resolved = resolveScanMatches(matches, merged.resolveOverlaps);
stats.matchesReturned = resolved.length;
return { matches: resolved, stats };
}
export async function pronunciationReplaceWithModel(model, text, terms, options = {}) {
const merged = mergeReplaceOptions(options);
const rawScan = await pronunciationScanWithModel(model, text, terms, {
...merged,
resolveOverlaps: "all",
});
const rawMatches = [...rawScan.matches];
const candidates = convertMatchesToPatchCandidates(rawMatches, text, merged);
const { survivors: deduped, discarded: duplicateDiscarded } = dedupePatchCandidates(candidates);
const { survivors: disambiguated, discarded: ambiguousDiscarded } = markAmbiguous(deduped, merged);
const { selected, discarded: overlapDiscarded } = selectNonOverlapping(disambiguated, merged);
const { text: finalText, patches: appliedSelected } = applyPatches(text, selected);
const applied = appliedSelected.filter((patch) => patch.status === "applied" || (patch.status === "unchanged" && merged.includeUnchanged));
const discarded = merged.includeDiscarded
? [...duplicateDiscarded, ...ambiguousDiscarded, ...overlapDiscarded].sort((left, right) => left.startChar - right.startChar ||
left.endChar - right.endChar ||
right.score - left.score)
: [];
const patches = [...applied, ...discarded].sort((left, right) => {
const leftStatusPriority = left.status === "applied" || left.status === "unchanged" ? 0 : 1;
const rightStatusPriority = right.status === "applied" || right.status === "unchanged" ? 0 : 1;
return (left.startChar - right.startChar ||
leftStatusPriority - rightStatusPriority ||
right.score - left.score);
});
return {
originalText: text,
text: finalText,
applied,
discarded,
patches,
stats: {
...rawScan.stats,
rawMatches: rawMatches.length,
dedupedMatches: deduped.length,
ambiguousDiscarded: ambiguousDiscarded.length,
overlapDiscarded: overlapDiscarded.length,
duplicateDiscarded: duplicateDiscarded.length,
appliedCount: appliedSelected.filter((patch) => patch.status === "applied").length,
unchangedCount: appliedSelected.filter((patch) => patch.status === "unchanged").length,
},
rawMatches: merged.keepScanMatches ? rawMatches : null,
};
}
export const mergeScanOptions = (options = {}) => ({
...DEFAULT_SCAN_OPTIONS,
...options,
});
export const mergeReplaceOptions = (options = {}) => ({
...DEFAULT_REPLACE_OPTIONS,
...options,
});
const emptyScanStats = (tokenCount) => ({
tokenCount,
windowCount: 0,
candidateVariantsConsidered: 0,
candidateVariantsVerified: 0,
matchesReturned: 0,
rejectedByLength: 0,
rejectedByInputLimit: 0,
rejectedByQgram: 0,
rejectedByDistance: 0,
});
const normalizeForMatch = (text) => {
const normalized = text.normalize("NFKC");
const mapped = Array.from(normalized)
.map((ch) => {
if (APOSTROPHE_CHARS.has(ch))
return "'";
if (DASH_CHARS.has(ch))
return "-";
return ch;
})
.join("")
.toLocaleLowerCase("und")
.normalize("NFKD");
const stripped = Array.from(mapped)
.filter((ch) => !/\p{M}/u.test(ch))
.join("");
return stripped.replace(/\s+/gu, " ").trim();
};
const compactSurface = (text) => text.replace(/ /gu, "").replace(/-/gu, "").replace(/'/gu, "");
const isWordChar = (ch) => /\p{L}|\p{N}/u.test(ch);
const characterUnitCount = (text) => toCodePoints(text).filter((codePoint) => isWordChar(codePoint.ch)).length;
const toCodePoints = (text) => {
const result = [];
let codeUnitOffset = 0;
let charIndex = 0;
while (codeUnitOffset < text.length) {
const code = text.codePointAt(codeUnitOffset);
const ch = String.fromCodePoint(code);
result.push({
ch,
charIndex,
codeUnitStart: codeUnitOffset,
codeUnitEnd: codeUnitOffset + ch.length,
});
codeUnitOffset += ch.length;
charIndex += 1;
}
return result;
};
const tokenizeWithOffsets = (text) => {
const codePoints = toCodePoints(text);
const tokens = [];
let idx = 0;
while (idx < codePoints.length) {
if (!isWordChar(codePoints[idx].ch)) {
idx += 1;
continue;
}
const start = idx;
idx += 1;
while (idx < codePoints.length) {
const current = codePoints[idx];
if (isWordChar(current.ch)) {
idx += 1;
continue;
}
if (WORD_JOINERS.has(current.ch) &&
idx + 1 < codePoints.length &&
idx > start &&
isWordChar(codePoints[idx - 1].ch) &&
isWordChar(codePoints[idx + 1].ch)) {
idx += 1;
continue;
}
break;
}
const slice = codePoints.slice(start, idx);
tokens.push({
rawText: text.slice(slice[0].codeUnitStart, slice[slice.length - 1].codeUnitEnd),
startChar: slice[0].charIndex,
endChar: slice[slice.length - 1].charIndex + 1,
startCodeUnit: slice[0].codeUnitStart,
endCodeUnit: slice[slice.length - 1].codeUnitEnd,
});
}
return tokens;
};
const prepareTokens = async (text, model, options, phoneEncoder) => {
const tokenCache = new Map();
const prepared = [];
for (const token of tokenizeWithOffsets(text)) {
const normText = normalizeForMatch(token.rawText);
const cacheKey = normText || token.rawText;
let cached = tokenCache.get(cacheKey);
if (!cached) {
const aligned = await phonemizeTextAligned(normText || token.rawText, model);
cached = {
phoneTokens: aligned.phoneTokens,
phones: aligned.phoneTokens.map((phone) => phoneEncoder.encode(phone)),
charIndexes: aligned.charIndexes,
};
tokenCache.set(cacheKey, cached);
}
prepared.push({
rawText: token.rawText,
normText,
startChar: token.startChar,
endChar: token.endChar,
startCodeUnit: token.startCodeUnit,
endCodeUnit: token.endCodeUnit,
phones: [...cached.phones],
phoneTokens: [...cached.phoneTokens],
charPhones: buildTokenCharPhones(token.rawText, normText, cached.phones, cached.phoneTokens, cached.charIndexes),
});
}
return prepared;
};
const buildCharacterUnits = (text, tokens) => {
const codePoints = toCodePoints(text);
const units = [];
let tokenIndex = 0;
for (const codePoint of codePoints) {
if (!isWordChar(codePoint.ch))
continue;
while (tokenIndex < tokens.length && codePoint.charIndex >= tokens[tokenIndex].endChar) {
tokenIndex += 1;
}
if (tokenIndex >= tokens.length)
break;
if (codePoint.charIndex < tokens[tokenIndex].startChar ||
codePoint.charIndex >= tokens[tokenIndex].endChar) {
continue;
}
units.push({
startChar: codePoint.charIndex,
endChar: codePoint.charIndex + 1,
startCodeUnit: codePoint.codeUnitStart,
endCodeUnit: codePoint.codeUnitEnd,
tokenIndex,
});
}
return units;
};
const compileVariants = async (terms, model, options, phoneEncoder, qgramEncoder, maxInputLen) => {
const variants = [];
const byTokenCount = new Map();
const indexByTokenCount = new Map();
let rejectedByInputLimit = 0;
let variantId = 0;
for (const rawTerm of terms) {
const term = coerceTerm(rawTerm);
const surfaces = [[term.text, null]];
for (const alias of term.aliases) {
surfaces.push([alias, alias]);
}
for (const [surfaceText, aliasText] of surfaces) {
const surfaceNorm = normalizeForMatch(surfaceText);
const tokenCount = options.spanUnit === "character"
? Math.max(1, characterUnitCount(surfaceText))
: tokenizeWithOffsets(surfaceText).length || Math.max(1, surfaceNorm.split(" ").filter(Boolean).length);
const pronunciationInputs = term.pronunciations.length > 0
? term.pronunciations.slice(0, options.maxTermPronunciations)
: [null];
for (const pronunciationInput of pronunciationInputs) {
if (pronunciationInput == null &&
maxInputLen != null &&
estimatePredictorInputLength(surfaceNorm) > maxInputLen) {
rejectedByInputLimit += 1;
continue;
}
const phoneTokens = pronunciationInput == null
? await phonemizeText(surfaceNorm, model)
: parseExplicitPronunciation(pronunciationInput);
const encoded = phoneTokens.map((phone) => phoneEncoder.encode(phone));
const thresholdLength = options.thresholdBasis === "phonemes" ? encoded.length : surfaceNorm.length;
const thresholdK = effectiveThreshold(thresholdLength, options.maxDistanceRatio, options.minDistance, options.maxDistance, options.allowShortFuzzy);
const qgramFreq = qgramFrequency(encoded, options.qgramSize, qgramEncoder);
const variant = {
variantId,
termId: term.id ?? null,
termText: term.text,
canonical: term.canonical,
aliasText,
metadata: term.metadata ?? null,
tokenCount,
surfaceNorm,
surfaceCompact: compactSurface(surfaceNorm),
phones: encoded,
phoneTokens,
phoneLen: encoded.length,
thresholdK,
qgramFreq,
pronunciationValue: pronunciationInput == null ? [...phoneTokens] : pronunciationInput,
};
variants.push(variant);
if (!byTokenCount.has(tokenCount))
byTokenCount.set(tokenCount, []);
byTokenCount.get(tokenCount).push(variant);
if (!indexByTokenCount.has(tokenCount))
indexByTokenCount.set(tokenCount, new Map());
for (const [qgramId, qgramCount] of qgramFreq.entries()) {
if (!indexByTokenCount.get(tokenCount).has(qgramId)) {
indexByTokenCount.get(tokenCount).set(qgramId, []);
}
indexByTokenCount.get(tokenCount).get(qgramId).push([variantId, qgramCount]);
}
variantId += 1;
}
}
}
return { variants, byTokenCount, indexByTokenCount, rejectedByInputLimit };
};
const scanCompiled = (text, tokens, compiled, options, qgramEncoder) => {
const stats = emptyScanStats(tokens.length);
if (compiled.variants.length === 0 || tokens.length === 0) {
return { matches: [], stats };
}
const tokenCounts = [...compiled.byTokenCount.keys()].sort((a, b) => a - b);
const variantById = new Map(compiled.variants.map((variant) => [variant.variantId, variant]));
const lengths = windowLengths(tokenCounts, options);
const rawMatches = [];
for (let startToken = 0; startToken < tokens.length; startToken++) {
for (const windowLength of lengths) {
const endToken = startToken + windowLength;
if (endToken > tokens.length)
continue;
stats.windowCount = (stats.windowCount ?? 0) + 1;
const window = buildWindow(text, tokens, startToken, endToken);
const relevantCounts = candidateTokenBuckets(windowLength, tokenCounts, options);
if (relevantCounts.length === 0)
continue;
const lengthOkIds = new Set();
for (const count of relevantCounts) {
for (const variant of compiled.byTokenCount.get(count) ?? []) {
if (Math.abs(window.phones.length - variant.phoneLen) > variant.thresholdK) {
stats.rejectedByLength = (stats.rejectedByLength ?? 0) + 1;
continue;
}
lengthOkIds.add(variant.variantId);
}
}
if (lengthOkIds.size === 0)
continue;
stats.candidateVariantsConsidered = (stats.candidateVariantsConsidered ?? 0) + lengthOkIds.size;
const windowQfreq = qgramFrequency(window.phones, options.qgramSize, qgramEncoder);
const candidateOverlap = new Map();
for (const count of relevantCounts) {
const postings = compiled.indexByTokenCount.get(count) ?? new Map();
for (const [qgramId, windowCount] of windowQfreq.entries()) {
for (const [variantId, termCount] of postings.get(qgramId) ?? []) {
if (!lengthOkIds.has(variantId))
continue;
candidateOverlap.set(variantId, (candidateOverlap.get(variantId) ?? 0) + Math.min(windowCount, termCount));
}
}
}
const verifiedIds = [];
for (const variantId of [...lengthOkIds].sort((a, b) => a - b)) {
const variant = variantById.get(variantId);
const required = requiredOverlap(variant.phoneLen, window.phones.length, options.qgramSize, variant.thresholdK);
if ((candidateOverlap.get(variantId) ?? 0) < required) {
stats.rejectedByQgram = (stats.rejectedByQgram ?? 0) + 1;
continue;
}
verifiedIds.push(variantId);
}
if (verifiedIds.length === 0)
continue;
stats.candidateVariantsVerified = (stats.candidateVariantsVerified ?? 0) + verifiedIds.length;
for (const variantId of verifiedIds) {
const variant = variantById.get(variantId);
const distance = verifyDistance(variant.phones, window.phones, variant.thresholdK, options.verifier);
if (distance == null) {
stats.rejectedByDistance = (stats.rejectedByDistance ?? 0) + 1;
continue;
}
const phonemeSimilarity = similarity(distance, variant.phones.length, window.phones.length);
const textDistance = levenshteinDistance(variant.surfaceCompact, window.surfaceCompact);
const textSimilarity = similarity(textDistance, variant.surfaceCompact.length, window.surfaceCompact.length);
const score = options.scoring === "phoneme"
? phonemeSimilarity
: options.phonemeWeight * phonemeSimilarity + options.textWeight * textSimilarity;
if (score < options.minScore)
continue;
rawMatches.push({
termId: variant.termId,
termText: variant.termText,
canonical: variant.canonical,
aliasText: variant.aliasText,
matchedText: window.matchedText,
startChar: window.startChar,
endChar: window.endChar,
startToken: window.startToken,
endToken: window.endToken,
score,
phonemeDistance: distance,
phonemeThreshold: variant.thresholdK,
phonemeSimilarity,
textDistance,
textSimilarity,
termPronunciation: options.returnPhonemes ? [...variant.phoneTokens] : null,
matchedPronunciation: options.returnPhonemes ? [...window.phoneTokens] : null,
metadata: variant.metadata,
});
}
}
}
return { matches: dedupeScanMatches(rawMatches), stats };
};
const scanCompiledByCharacters = async (text, tokens, compiled, options, qgramEncoder, model, phoneEncoder, maxInputLen) => {
const stats = emptyScanStats(tokens.length);
const charUnits = buildCharacterUnits(text, tokens);
if (compiled.variants.length === 0 || charUnits.length === 0) {
return { matches: [], stats };
}
const tokenCounts = [...compiled.byTokenCount.keys()].sort((a, b) => a - b);
const variantById = new Map(compiled.variants.map((variant) => [variant.variantId, variant]));
const lengths = windowLengths(tokenCounts, options);
const rawMatches = [];
const windowCache = new Map();
// Approximate phones per character unit, sliced out of each token's
// alignment buckets. They let us reject the vast majority of candidate
// windows with pure-JS filters before paying for a real G2P inference.
const unitPhones = charUnits.map((unit) => {
const token = tokens[unit.tokenIndex];
if (!token.charPhones)
return null;
return token.charPhones[unit.startChar - token.startChar] ?? null;
});
const prefixPhoneCounts = [0];
const prefixUnmappable = [0];
for (let idx = 0; idx < charUnits.length; idx += 1) {
const phones = unitPhones[idx];
prefixPhoneCounts.push(prefixPhoneCounts[idx] + (phones ? phones.length : 0));
prefixUnmappable.push(prefixUnmappable[idx] + (phones ? 0 : 1));
}
// Concatenated per-character phones approximate the true G2P output of the
// window text, so the prefilter widens every variant threshold by this slack
// before discarding a window without verification.
const approxSlackFor = (variant) => Math.max(2, Math.ceil(variant.phoneLen * 0.25));
for (let startUnit = 0; startUnit < charUnits.length; startUnit += 1) {
for (const windowLength of lengths) {
const endUnit = startUnit + windowLength;
if (endUnit > charUnits.length)
continue;
stats.windowCount = (stats.windowCount ?? 0) + 1;
const firstUnit = charUnits[startUnit];
const lastUnit = charUnits[endUnit - 1];
const windowText = text.slice(firstUnit.startCodeUnit, lastUnit.endCodeUnit);
const windowNorm = normalizeForMatch(windowText);
if (maxInputLen != null &&
estimatePredictorInputLength(windowNorm || windowText) > maxInputLen) {
stats.rejectedByInputLimit = (stats.rejectedByInputLimit ?? 0) + 1;
continue;
}
const relevantCounts = candidateTokenBuckets(windowLength, tokenCounts, options);
if (relevantCounts.length === 0)
continue;
// Alignment-based prefilter: only windows that plausibly match some
// variant (under slackened thresholds) are sent to the predictor.
const windowMappable = prefixUnmappable[endUnit] - prefixUnmappable[startUnit] === 0;
if (windowMappable) {
const approxLen = prefixPhoneCounts[endUnit] - prefixPhoneCounts[startUnit];
let approxLengthRejected = 0;
let approxQgramRejected = 0;
let approxDistanceRejected = 0;
const lengthOkVariants = [];
for (const count of relevantCounts) {
for (const variant of compiled.byTokenCount.get(count) ?? []) {
if (Math.abs(approxLen - variant.phoneLen) > variant.thresholdK + approxSlackFor(variant)) {
approxLengthRejected += 1;
continue;
}
lengthOkVariants.push(variant);
}
}
let plausible = false;
if (lengthOkVariants.length > 0) {
const approxPhones = [];
for (let unit = startUnit; unit < endUnit; unit += 1) {
approxPhones.push(...unitPhones[unit]);
}
const approxQfreq = qgramFrequency(approxPhones, options.qgramSize, qgramEncoder);
for (const variant of lengthOkVariants) {
const slackK = variant.thresholdK + approxSlackFor(variant);
const required = requiredOverlap(variant.phoneLen, approxPhones.length, options.qgramSize, slackK);
if (qgramOverlap(approxQfreq, variant.qgramFreq) < required) {
approxQgramRejected += 1;
continue;
}
if (verifyDistance(variant.phones, approxPhones, slackK, options.verifier) == null) {
approxDistanceRejected += 1;
continue;
}
plausible = true;
break;
}
}
if (!plausible) {
stats.candidateVariantsConsidered =
(stats.candidateVariantsConsidered ?? 0) + lengthOkVariants.length;
stats.rejectedByLength = (stats.rejectedByLength ?? 0) + approxLengthRejected;
stats.rejectedByQgram = (stats.rejectedByQgram ?? 0) + approxQgramRejected;
stats.rejectedByDistance = (stats.rejectedByDistance ?? 0) + approxDistanceRejected;
continue;
}
}
const window = await buildCharacterWindow(text, charUnits, startUnit, endUnit, model, phoneEncoder, windowCache, maxInputLen);
if (!window) {
stats.rejectedByInputLimit = (stats.rejectedByInputLimit ?? 0) + 1;
continue;
}
const lengthOkIds = new Set();
for (const count of relevantCounts) {
for (const variant of compiled.byTokenCount.get(count) ?? []) {
if (Math.abs(window.phones.length - variant.phoneLen) > variant.thresholdK) {
stats.rejectedByLength = (stats.rejectedByLength ?? 0) + 1;
continue;
}
lengthOkIds.add(variant.variantId);
}
}
if (lengthOkIds.size === 0)
continue;
stats.candidateVariantsConsidered = (stats.candidateVariantsConsidered ?? 0) + lengthOkIds.size;
const windowQfreq = qgramFrequency(window.phones, options.qgramSize, qgramEncoder);
const candidateOverlap = new Map();
for (const count of relevantCounts) {
const postings = compiled.indexByTokenCount.get(count) ?? new Map();
for (const [qgramId, windowCount] of windowQfreq.entries()) {
for (const [variantId, termCount] of postings.get(qgramId) ?? []) {
if (!lengthOkIds.has(variantId))
continue;
candidateOverlap.set(variantId, (candidateOverlap.get(variantId) ?? 0) + Math.min(windowCount, termCount));
}
}
}
const verifiedIds = [];
for (const variantId of [...lengthOkIds].sort((a, b) => a - b)) {
const variant = variantById.get(variantId);
const required = requiredOverlap(variant.phoneLen, window.phones.length, options.qgramSize, variant.thresholdK);
if ((candidateOverlap.get(variantId) ?? 0) < required) {
stats.rejectedByQgram = (stats.rejectedByQgram ?? 0) + 1;
continue;
}
verifiedIds.push(variantId);
}
if (verifiedIds.length === 0)
continue;
stats.candidateVariantsVerified = (stats.candidateVariantsVerified ?? 0) + verifiedIds.length;
for (const variantId of verifiedIds) {
const variant = variantById.get(variantId);
const distance = verifyDistance(variant.phones, window.phones, variant.thresholdK, options.verifier);
if (distance == null) {
stats.rejectedByDistance = (stats.rejectedByDistance ?? 0) + 1;
continue;
}
const phonemeSimilarity = similarity(distance, variant.phones.length, window.phones.length);
const textDistance = levenshteinDistance(variant.surfaceCompact, window.surfaceCompact);
const textSimilarity = similarity(textDistance, variant.surfaceCompact.length, window.surfaceCompact.length);
const score = options.scoring === "phoneme"
? phonemeSimilarity
: options.phonemeWeight * phonemeSimilarity + options.textWeight * textSimilarity;
if (score < options.minScore)
continue;
rawMatches.push({
termId: variant.termId,
termText: variant.termText,
canonical: variant.canonical,
aliasText: variant.aliasText,
matchedText: window.matchedText,
startChar: window.startChar,
endChar: window.endChar,
startToken: window.startToken,
endToken: window.endToken,
score,
phonemeDistance: distance,
phonemeThreshold: variant.thresholdK,
phonemeSimilarity,
textDistance,
textSimilarity,
termPronunciation: options.returnPhonemes ? [...variant.phoneTokens] : null,
matchedPronunciation: options.returnPhonemes ? [...window.phoneTokens] : null,
metadata: variant.metadata,
});
}
}
}
return { matches: dedupeScanMatches(rawMatches), stats };
};
const windowLengths = (termTokenCounts, options) => {
if (termTokenCounts.length === 0)
return [];
if (options.wordBoundaryMode === "strict") {
return [...new Set(termTokenCounts.filter((count) => count > 0))].sort((a, b) => a - b);
}
const minCount = Math.max(1, Math.min(...termTokenCounts) - options.tokenSlack);
const maxCount = Math.max(...termTokenCounts) + options.tokenSlack;
const result = [];
for (let value = minCount; value <= maxCount; value += 1) {
result.push(value);
}
return result;
};
const candidateTokenBuckets = (windowLength, termTokenCounts, options) => {
if (options.wordBoundaryMode === "strict") {
return termTokenCounts.includes(windowLength) ? [windowLength] : [];
}
return termTokenCounts.filter((count) => Math.abs(count - windowLength) <= options.tokenSlack);
};
const buildWindow = (text, tokens, startToken, endToken) => {
const selected = tokens.slice(startToken, endToken);
const surfaceNorm = selected.map((token) => token.normText).filter(Boolean).join(" ");
const phones = [];
const phoneTokens = [];
for (const token of selected) {
phones.push(...token.phones);
phoneTokens.push(...token.phoneTokens);
}
return {
startToken,
endToken,
startChar: selected[0].startChar,
endChar: selected[selected.length - 1].endChar,
matchedText: text.slice(selected[0].startCodeUnit, selected[selected.length - 1].endCodeUnit),
surfaceNorm,
surfaceCompact: compactSurface(surfaceNorm),
phones,
phoneTokens,
};
};
const buildCharacterWindow = async (text, charUnits, startUnit, endUnit, model, phoneEncoder, cache, maxInputLen) => {
const first = charUnits[startUnit];
const last = charUnits[endUnit - 1];
const matchedText = text.slice(first.startCodeUnit, last.endCodeUnit);
const surfaceNorm = normalizeForMatch(matchedText);
const cacheKey = surfaceNorm || matchedText;
if (!cache.has(cacheKey)) {
if (maxInputLen != null && estimatePredictorInputLength(surfaceNorm || matchedText) > maxInputLen) {
cache.set(cacheKey, null);
return null;
}
const phoneTokens = await phonemizeText(surfaceNorm || matchedText, model);
cache.set(cacheKey, {
surfaceNorm,
phoneTokens,
phones: phoneTokens.map((phone) => phoneEncoder.encode(phone)),
});
}
const cached = cache.get(cacheKey);
if (!cached)
return null;
return {
startToken: first.tokenIndex,
endToken: last.tokenIndex + 1,
startChar: first.startChar,
endChar: last.endChar,
matchedText,
surfaceNorm: cached.surfaceNorm,
surfaceCompact: compactSurface(cached.surfaceNorm),
phones: [...cached.phones],
phoneTokens: [...cached.phoneTokens],
};
};
const resolvePredictorMaxInputLen = (model) => {
if (typeof model.getMaxInputLen === "function") {
const value = model.getMaxInputLen();
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
return Math.trunc(value);
}
}
const carrier = model;
if (typeof carrier.maxInputLen === "number" &&
Number.isFinite(carrier.maxInputLen) &&
carrier.maxInputLen > 0) {
return Math.trunc(carrier.maxInputLen);
}
const optionValue = carrier.options?.maxInputLen;
if (typeof optionValue === "number" && Number.isFinite(optionValue) && optionValue > 0) {
return Math.trunc(optionValue);
}
return null;
};
const estimatePredictorInputLength = (text) => {
const sequence = splitTextToJamo(text);
return sequence.tokens.length > 0 ? sequence.tokens.length : 1;
};
const requiredOverlap = (termLen, windowLen, q, thresholdK) => Math.max(0, Math.max(termLen, windowLen) - q + 1 - thresholdK * q);
const verifyDistance = (pattern, text, thresholdK, verifier) => {
if (Math.abs(pattern.length - text.length) > thresholdK)
return null;
if (verifier === "myers") {
const distance = myersDistance(pattern, text);
return distance <= thresholdK ? distance : null;
}
if (verifier === "auto" && pattern.length <= 64) {
const distance = myersDistance(pattern, text);
return distance <= thresholdK ? distance : null;
}
return ukkonenDistance(pattern, text, thresholdK);
};
const myersDistance = (pattern, text) => {
const m = pattern.length;
if (m === 0)
return text.length;
if (text.length === 0)
return m;
const peq = new Map();
for (let idx = 0; idx < pattern.length; idx += 1) {
peq.set(pattern[idx], (peq.get(pattern[idx]) ?? 0n) | (1n << BigInt(idx)));
}
const mask = (1n << BigInt(m)) - 1n;
let pv = mask;
let mv = 0n;
let score = m;
const highBit = 1n << BigInt(m - 1);
for (const symbol of text) {
const eq = peq.get(symbol) ?? 0n;
const xv = eq | mv;
const xh = (((eq & pv) + pv) ^ pv) | eq;
let ph = mv | (~(xh | pv) & mask);
let mh = pv & xh;
if ((ph & highBit) !== 0n) {
score += 1;
}
else if ((mh & highBit) !== 0n) {
score -= 1;
}
ph = ((ph << 1n) | 1n) & mask;
mh = (mh << 1n) & mask;
pv = (mh | (~(xv | ph) & mask)) & mask;
mv = ph & xv;
}
return score;
};
const ukkonenDistance = (pattern, text, thresholdK) => {
const m = pattern.length;
const n = text.length;
if (m === 0)
return n <= thresholdK ? n : null;
if (n === 0)
return m <= thresholdK ? m : null;
const inf = thresholdK + 1;
let prev = Array.from({ length: m + 1 }, (_, idx) => idx);
for (let i = 1; i <= n; i += 1) {
const curr = new Array(m + 1).fill(inf);
const lo = Math.max(1, i - thresholdK);
const hi = Math.min(m, i + thresholdK);
if (lo === 1)
curr[0] = i;
for (let j = lo; j <= hi; j += 1) {
const cost = pattern[j - 1] === text[i - 1] ? 0 : 1;
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
}
const bandMin = curr.slice(lo, hi + 1).reduce((best, value) => Math.min(best, value), inf);
if (bandMin > thresholdK)
return null;
prev = curr;
}
return prev[m] <= thresholdK ? prev[m] : null;
};
const levenshteinDistance = (left, right) => {
if (left === right)
return 0;
if (!left)
return right.length;
if (!right)
return left.length;
let prev = Array.from({ length: right.length + 1 }, (_, idx) => idx);
for (let i = 0; i < left.length; i += 1) {
const curr = [i + 1];
for (let j = 0; j < right.length; j += 1) {
const cost = left[i] === right[j] ? 0 : 1;
curr.push(Math.min(prev[j + 1] + 1, curr[j] + 1, prev[j] + cost));
}
prev = curr;
}
return prev[prev.length - 1];
};
const similarity = (distance, leftLen, rightLen) => 1 - distance / Math.max(leftLen, rightLen, 1);
const dedupeScanMatches = (matches) => {
const winners = new Map();
for (const match of matches) {
const key = `${match.startChar}:${match.endChar}:${match.canonical}`;
const previous = winners.get(key);
if (!previous || compareScanMatch(match, previous) < 0) {
winners.set(key, match);
}
}
return [...winners.values()].sort((left, right) => left.startChar - right.startChar ||
left.endChar - right.endChar ||
right.score - left.score);
};
const resolveScanMatches = (matches, mode) => {
if (mode === "all") {
return [...matches].sort((left, right) => left.startChar - right.startChar ||
left.endChar - right.endChar ||
right.score - left.score);
}
if (mode === "per_term_best") {
const winners = new Map();
for (const match of matches) {
const previous = winners.get(match.canonical);
if (!previous || compareScanMatch(match, previous) < 0) {
winners.set(match.canonical, match);
}
}
return [...winners.values()].sort((left, right) => left.startChar - right.startChar ||
left.endChar - right.endChar ||
right.score - left.score);
}
const ordered = [...matches].sort((left, right) => {
const lengthDelta = (right.endChar - right.startChar) - (left.endChar - left.startChar);
return (right.score - left.score ||
left.phonemeDistance - right.phonemeDistance ||
lengthDelta ||
left.startChar - right.startChar);
});
const chosen = [];
for (const match of ordered) {
if (chosen.some((existing) => overlaps(match, existing)))
continue;
chosen.push(match);
}
return chosen.sort((left, right) => left.startChar - right.startChar ||
left.endChar - right.endChar ||
right.score - left.score);
};
const convertMatchesToPatchCandidates = (matches, originalText, options) => {
const boundaries = buildCharBoundaries(originalText);
const candidates = [];
for (const match of matches) {
let replacementText = resolveReplacementText(match, options);
if (!replacementText)
continue;
replacementText = applyCaseStrategy(replacementText, match.matchedText, options);
const sourceText = sliceByCharRange(originalText, boundaries, match.startChar, match.endChar);
candidates.push({
...match,
status: "applied",
discardReason: null,
replacementText,
outputStartChar: null,
outputEndChar: null,
changed: sourceText !== replacementText,
deltaChars: charLength(replacementText) - (match.endChar - match.startChar),
});
}
return candidates;
};
const resolveReplacementText = (match, options) => {
if (options.replacementSource === "term_text")
return match.termText;
if (options.replacementSource === "alias_text")
return match.aliasText ?? match.termText;
return match.canonical || match.termText;
};
const applyCaseStrategy = (replacementText, matchedText, options) => {
if (options.caseStrategy !== "match_simple")
return replacementText;
if (isAllUpper(matchedText))
return replacementText.toUpperCase();
if (isTitleCase(matchedText))
return titleCaseWords(replacementText);
return replacementText;
};
const isAllUpper = (text) => {
const letters = Array.from(text).filter((ch) => /\p{L}/u.test(ch));
return letters.length > 0 && letters.every((ch) => ch === ch.toUpperCase());
};
const isTitleCase = (text) => {
const words = text.trim().split(/\s+/u).filter(Boolean);
return words.length > 0 && words.every((word) => isTitleCaseWord(word));
};
const isTitleCaseWord = (word) => {
const parts = word.split(/([-'])/u).filter(Boolean);
let sawCased = false;
for (const part of parts) {
if (part === "-" || part === "'")
continue;
const letters = Array.from(part).filter((ch) => /\p{L}/u.test(ch));
if (letters.length === 0)
continue;
sawCased = true;
if (letters[0] !== letters[0].toUpperCase())
return false;
if (letters.slice(1).some((ch) => ch !== ch.toLowerCase()))
return false;
}
return sawCased;
};
const titleCaseWords = (text) => text
.split(" ")
.map((word) => word
.split("-")
.map((part) => part
.split("'")
.map((segment) => segment ? segment[0].toUpperCase() + segment.slice(1).toLowerCase() : "")
.join("'"))
.join("-"))
.join(" ");
const dedupePatchCandidates = (candidates) => {
const winners = new Map();
const discarded = [];
for (const candidate of candidates) {
const key = `${candidate.startChar}:${candidate.endChar}:${candidate.canonical}:${candidate.replacementText}`;
const previous = winners.get(key);
if (!previous) {
winners.set(key, candidate);
continue;
}
const better = comparePatch(candidate, previous) < 0 ? candidate : previous;
const loser = better === candidate ? previous : candidate;
winners.set(key, better);
discarded.push({
...loser,
status: "discarded_duplicate",
discardReason: "duplicate_of_better_variant",
outputStartChar: null,
outputEndChar: null,
});
}
return { survivors: [...winners.values()], discarded };
};
const markAmbiguous = (candidates, options) => {
const groups = new Map();
for (const candidate of candidates) {
const key = `${candidate.startChar}:${candidate.endChar}`;
if (!groups.has(key))
groups.set(key, []);
groups.get(key).push(candidate);
}
const survivors = [];
const discarded = [];
for (const group of groups.values()) {
const ordered = [...group].sort(comparePatch);
if (ordered.length < 2 ||
ordered[0].canonical === ordered[1].canonical ||
Math.abs(ordered[0].score - ordered[1].score) >= options.ambiguityMargin) {
survivors.push(...ordered);
continue;
}
if (options.ambiguousPolicy === "keep_best") {
survivors.push(ordered[0]);
for (const loser of ordered.slice(1)) {
discarded.push({
...loser,
status: "discarded_ambiguous",
discardReason: "same_span_competing_canonicals",
outputStartChar: null,
outputEndChar: null,
});
}
continue;
}
for (const loser of ordered) {
discarded.push({
...loser,
status: "discarded_ambiguous",
discardReason: "same_span_competing_canonicals",
outputStartChar: null,
outputEndChar: null,
});
}
}
return { survivors, discarded };
};
const selectNonOverlapping = (candidates, options) => {
if (candidates.length === 0)
return { selected: [], discarded: [] };
if (options.conflictPolicy === "greedy_left_to_right")
return selectGreedy(candidates);
if (options.conflictPolicy === "error") {
const ordered = [...candidates].sort((left, right) => left.startChar - right.startChar || left.endChar - right.endChar);
for (let idx = 1; idx < ordered.length; idx += 1) {
if (overlaps(ordered[idx - 1], ordered[idx])) {
throw new Error("Overlapping pronunciation replacements remain after ambiguity resolution");
}
}
return { selected: ordered, discarded: [] };
}
return selectWeightedInterval(candidates);
};
const selectGreedy = (candidates) => {
const ordered = [...candidates].sort((left, right) => {
const lengthDelta = (right.endChar - right.startChar) - (left.endChar - left.startChar);
return left.startChar - right.startChar || right.score - left.score || lengthDelta;
});
const selected = [];
const discarded = [];
for (const candidate of ordered) {
if (selected.some((existing) => overlaps(candidate, existing))) {
discarded.push({
...candidate,
status: "discarded_overlap",
discardReason: "lost_to_higher_value_non_overlapping_set",
outputStartChar: null,
outputEndChar: null,
});
continue;
}
selected.push(candidate);
}
return {
selected: selected.sort((left, right) => left.startChar - right.startChar || left.endChar - right.endChar),
discarded,
};
};
const selectWeightedInterval = (candidates) => {
const ordered = [...candidates].sort((left, right) => {
const lengthDelta = (right.endChar - right.startChar) - (left.endChar - left.startChar);
return (left.endChar - right.endChar ||
left.startChar - right.startChar ||
right.score - left.score ||
lengthDelta);
});
const endPositions = ordered.map((candidate) => candidate.endChar);
const predecessors = [];
for (let idx = 0; idx < ordered.length; idx += 1) {
let predecessor = bisectRight(endPositions, ordered[idx].startChar) - 1;
while (predecessor >= 0 && overlaps(ordered[predecessor], ordered[idx])) {
predecessor -= 1;
}
predecessors.push(predecessor);
}
const states = Array.from({ length: ordered.length + 1 }, () => [0, 0, 0, 0]);
const takeFlags = new Array(ordered.length).fill(false);
for (let idx = 1; idx <= ordered.length; idx += 1) {
const candidate = ordered[idx - 1];
const pred = predecessors[idx - 1] + 1;
const take = addPatchValue(states[pred], candidate);
const skip = states[idx - 1];
if (compareState(take, skip) > 0) {
states[idx] = take;
takeFlags[idx - 1] = true;
}
else {
states[idx] = skip;
}
}
const selectedIndexes = new Set();
let idx = ordered.length;
while (idx > 0) {
const candidate = ordered[idx - 1];
const pred = predecessors[idx - 1] + 1;
const take = addPatchValue(states[pred], candidate);
if (takeFlags[idx - 1] && compareState(take, states[idx]) === 0) {
selectedIndexes.add(idx - 1);
idx = pred;
}
else {
idx -= 1;
}
}
const selected = ordered.filter((_, index) => selectedIndexes.has(index));
const discarded = ordered
.filter((_, index) => !selectedIndexes.has(index))
.map((candidate) => ({
...candidate,
status: "discarded_overlap",
discardReason: "lost_to_higher_value_non_overlapping_set",
outputStartChar: null,
outputEndChar: null,
}));
return { selected, discarded };
};
const addPatchValue = (state, patch) => [
state[0] + Math.round(patch.score * 1_000_000),
state[1] + (patch.endChar - patch.startChar),
state[2] - patch.phonemeDistance,
state[3] - 1,
];
const compareState = (left, right) => {
for (let idx = 0; idx < left.length; idx += 1) {
if (left[idx] !== right[idx])
return left[idx] > right[idx] ? 1 : -1;
}
return 0;
};
const applyPatches = (originalText, selected) => {
const ordered = [...selected].sort((left, right) => left.startChar - right.startChar || left.endChar - right.endChar);
const boundaries = buildCharBoundaries(originalText);
let cursorChar = 0;
let outputCharLength = 0;
const chunks = [];
const patches = [];
for (const patch of ordered) {
const untouched = sliceByCharRange(originalText, boundaries, cursorChar, patch.startChar);
chunks.push(untouched);
outputCharLength += charLength(untouched);
const outputStartChar = outputCharLength;
chunks.push(patch.replacementText);
outputCharLength += charLength(patch.replacementText);
const outputEndChar = outputCharLength;
patches.push({
...patch,
status: patch.changed ? "applied" : "unchanged",
discardReason: null,
outputStartChar,
outputEndChar,
});
cursorChar = patch.endChar;
}
chunks.push(sliceByCharRange(originalText, boundaries, cursorChar, boundaries.length - 1));
return { text: chunks.join(""), patches };
};
const coerceTerm = (term) => {
if (typeof term === "string") {
return {
id: "",
text: term,
canonical: term,
pronunciations: [],
aliases: [],
metadata: {},
};
}
return {
id: term.id ?? "",
text: term.text,
canonical: term.canonical ?? term.text,
pronunciations: term.pronunciations ?? [],
aliases: term.aliases ?? [],
metadata: term.metadata ?? {},
};
};
const phonemizeText = async (text, model) => {
const { phoneTokens } = await phonemizeTextAligned(text, model);
return phoneTokens;
};
const phonemizeTextAligned = async (text, model) => {
const normalized = normalizeForMatch(text);
const fallback = pseudoPhonesAligned(normalized);
if (!normalized)
return fallback;
try {
const result = await model.predict(normalized, {
splitDelimiter: null,
outputDelimiter: "",
preserveLiterals: "none",
});
// Some languages decode with literal separator tokens between phonemes.
// They carry no pronunciation signal, so drop them before matching.
const aligned = result.alignments.filter((alignment) => alignment.phoneme.trim() !== "");
if (aligned.length === 0)
return fallback;
return {
phoneTokens: aligned.map((alignment) => alignment.phoneme),
charIndexes: aligned.map((alignment) => alignment.charIndex),
};
}
catch {
return fallback;
}
};
/** Mirrors pseudoPhones but keeps the source character index per pseudo-phone. */
const pseudoPhonesAligned = (text) => {
const phoneTokens = [];
const charIndexes = [];
Array.from(text).forEach((ch, idx) => {
if (/\s/u.test(ch) || ch === "-" || ch === "'")
return;
phoneTokens.push(ch);
charIndexes.push(idx);
});
if (phoneTokens.length === 0) {
return { phoneTokens: ["<unk>"], charIndexes: [-1] };
}
return { phoneTokens, charIndexes };
};
/**
* Buckets a token's encoded phones by raw character offset using the
* predictor's normalized-input character alignments. Returns null when the
* raw→normalized offset mapping is not compositional (per-character
* normalization lengths fail to add up to the normalized string).
*/
const buildTokenCharPhones = (rawText, normText, phones, phoneTokens, charIndexes) => {
const rawChars = Array.from(rawText);
if (rawChars.length === 0)
return null;
const normLen = Array.from(normText).length;
// Spell-out mode: when every phone is literally its source character the
// predictor is naming letters, not pronouncing the word, so per-character
// slices would not approximate how a sub-span is actually pronounced.
const normChars = Array.from(normText);
let letterIdentity = phoneTokens.length > 0;
for (let idx = 0; idx < phoneTokens.length && letterIdentity; idx += 1) {
const charIndex = charIndexes[idx];
if (charIndex < 0 || charIndex >= normChars.length || phoneTokens[idx] !== normChars[charIndex]) {
letterIdentity = false;
}
}
if (letterIdentity)
return null;
// Trailing characters with no aligned phones usually mean the prediction was
// truncated; windows over the tail would look spuriously short.
let maxAligned = -1;
for (const charIndex of charIndexes) {
if (charIndex > maxAligned)
maxAligned = charIndex;
}
if (normLen - 1 - maxAligned >= 3)
return null;
const cum = [0];
for (const ch of rawChars) {
cum.push(cum[cum.length - 1] + Array.from(normalizeForMatch(ch)).length);
}
if (cum[cum.length - 1] !== normLen)
return null;
const buckets = rawChars.map(() => []);
for (let idx = 0; idx < phones.length; idx += 1) {
const charIndex = charIndexes[idx] ?? -1;
let target;
if (charIndex < 0) {
target = 0;
}
else if (charIndex >= normLen) {
target = rawChars.length - 1;
}
else {
let lo = 0;
let hi = rawChars.length - 1;
target = rawChars.length - 1;
while (lo <= hi) {
const mid = (lo + hi) >> 1;
if (cum[mid] <= charIndex && charIndex < cum[mid + 1]) {
target = mid;
break;
}
if (charIndex < cum[mid]) {
hi = mid - 1;
}
else {
lo = mid + 1;
}
}
}
buckets[target].push(phones[idx]);
}
return buckets;
};
const pseudoPhones = (text) => {
const compact = compactSurface(text);
return Array.from(compact).filter((ch) => !/\s/u.test(ch)).length > 0
? Array.from(compact).filter((ch) => !/\s/u.test(ch))
: ["<unk>"];
};
const parseExplicitPronunciation = (value) => {
if (typeof value === "string") {
const stripped = value.trim();
if (!stripped)
return ["<unk>"];
if (/\s/u.test(stripped))
return stripped.split(/\s+/u).filter(Boolean);
return pseudoPhones(stripped);
}
return value.filter(Boolean);
};
const effectiveThreshold = (length, maxDistanceRatio, minDistance, maxDistance, allowShortFuzzy) => {
let threshold = Math.max(Math.floor(length * maxDistanceRatio), minDistance);
if (maxDistance != null)
threshold = Math.min(threshold, maxDistance);
if (!allowShortFuzzy) {
if (length <= 3)
return 0;
if (length <= 6)
return Math.min(threshold, 1);
}
return threshold;
};
const qgramOverlap = (left, right) => {
const [small, large] = left.size <= right.size ? [left, right] : [right, left];
let overlap = 0;
for (const [qgramId, count] of small.entries()) {
const other = large.get(qgramId);
if (other != null)
overlap += Math.min(count, other);
}
return overlap;
};
const qgramFrequency = (sequence, q, encoder) => {
const freq = new Map();
if (q <= 0 || sequence.length < q)
return freq;
for (let idx = 0; idx <= sequence.length - q; idx += 1) {
const qgramId = encoder.encode(sequence.slice(idx, idx + q));
freq.set(qgramId, (freq.get(qgramId) ?? 0) + 1);
}
return freq;
};
const compareScanMatch = (left, right) => right.score - left.score ||
left.phonemeDistance - right.phonemeDistance ||
(right.textSimilarity ?? 0) - (left.textSimilarity ?? 0);
const comparePatch = (left, right) => right.score - left.score ||
left.phonemeDistance - right.phonemeDistance ||
(right.textSimilarity ?? 0) - (left.textSimilarity ?? 0);
const overlaps = (left, right) => !(left.endChar <= right.startChar || right.endChar <= left.startChar);
const buildCharBoundaries = (text) => {
const boundaries = [0];
let offset = 0;
while (offset < text.length) {
const code = text.codePointAt(offset);
const ch = String.fromCodePoint(code);
offset += ch.length;
boundaries.push(offset);
}
return boundaries;
};
const sliceByCharRange = (text, boundaries, startChar, endChar) => text.slice(boundaries[startChar] ?? 0, boundaries[endChar] ?? text.length);
const charLength = (text) => Array.from(text).length;
const bisectRight = (values, target) => {
let lo = 0;
let hi = values.length;
while (lo < hi) {
const mid = (lo + hi) >> 1;
if (target < values[mid]) {
hi = mid;
}
else {
lo = mid + 1;
}
}
return lo;
};
class PhoneEncoder {
constructor() {
this.nextId = 1;
this.mapping = new Map();
}
encode(phone) {
const existing = this.mapping.get(phone);
if (existing != null)
return existing;
const assigned = this.nextId;
this.mapping.set(phone, assigned);
this.nextId += 1;
return assigned;
}
}
class QGramEncoder {
constructor() {
this.nextId = 1;
this.mapping = new Map();
}
encode(qgram) {
const key = qgram.join(",");
const existing = this.mapping.get(key);
if (existing != null)
return existing;
const assigned = this.nextId;
this.mapping.set(key, assigned);
this.nextId += 1;
return assigned;
}
}
//# sourceMappingURL=pronunciation.js.map