phonemize/dist/core.mjs

Fast phonemizer with rule-based G2P prediction. Pure JavaScript implementation.

import{r as resolveJson,P as PUNCTUATION,i as ipaToArpabet,c as convertChineseTonesToArrows,_ as __assign}from"./utils-Bo_ZN3X7.mjs";import*as BLOCK_MAP from"./anyascii.json";function normalizeTag(n){return n.toLowerCase()}function tagCovers(n,s){var c=normalizeTag(n),l=normalizeTag(s);return c===l?!0:l.startsWith(c+`-`)}var LanguageRegistry=function(){function n(){this.processors=new Map,this.order=[]}return n.prototype.register=function(n){var s=this.processors.get(n.id);if(s){var c=this.order.indexOf(s);c===-1?this.order.push(n):this.order[c]=n}else this.order.push(n);this.processors.set(n.id,n)},n.prototype.unregister=function(n){var s=this.processors.get(n);return s?(this.processors.delete(n),this.order=this.order.filter(function(s){return s.id!==n}),!0):!1},n.prototype.getProcessor=function(n){return this.processors.get(n)},n.prototype.getProcessorsForLanguage=function(n){for(var s=normalizeTag(n),c=[],l=[],u=0,d=0,m=this.order;d<m.length;d++){for(var h=m[d],g=!1,_=0,v=0,y=h.supportedLanguages;v<y.length;v++){var b=y[v],x=normalizeTag(b);if(x===s){g=!0;break}tagCovers(b,n)&&x.length>_&&(_=x.length)}g?c.push(h):_>0&&l.push({proc:h,matchLen:_,order:u}),u++}return l.sort(function(n,s){return s.matchLen-n.matchLen||n.order-s.order}),c.concat(l.map(function(n){return n.proc}))},n.prototype.getAllProcessors=function(){return this.order.slice()},n.prototype.findBestProcessor=function(n,s){var c;if(s){var l=this.getProcessorsForLanguage(s);if(l.length>0)return l[0]}return(c=this.order[0])??null},n.prototype.predictPhonemes=function(n,s,c){var l=s;if(!l){var u=detectLanguage(n);u&&(l=u)}var d=this.findBestProcessor(n,l);if(!d)return null;var f=d.predict(n,l,c);return f&&f.trim()?f:null},n.prototype.clear=function(){this.processors.clear(),this.order=[]},n.prototype.getSupportedLanguages=function(){for(var n=new Set,s=0,c=this.order;s<c.length;s++)for(var l=c[s],u=0,d=l.supportedLanguages;u<d.length;u++){var f=d[u];n.add(f)}return Array.from(n)},n}(),languageRegistry=new LanguageRegistry,CHINESE_CHARS=/[一-龥]/,JAPANESE_CHARS=/[぀-ヿ]/,KOREAN_CHARS=/[가-힯]/,RUSSIAN_CHARS=/[Ѐ-ӿ]/,GERMAN_CHARS=/[äöüÄÖÜß]/,ARABIC_CHARS=/[؀-ۿ]/,THAI_CHARS=/[฀-๿]/;function detectLanguage(n){return CHINESE_CHARS.test(n)?`zh`:JAPANESE_CHARS.test(n)?`ja`:KOREAN_CHARS.test(n)?`ko`:RUSSIAN_CHARS.test(n)?`ru`:GERMAN_CHARS.test(n)?`de`:ARABIC_CHARS.test(n)?`ar`:THAI_CHARS.test(n)?`th`:null}function classifyChar(n,s){return s===void 0&&(s=`zh`),n>=19968&&n<=40959||n>=13312&&n<=19903||n>=63744&&n<=64255?s:n>=12352&&n<=12447||n>=12448&&n<=12543?`ja`:n>=44032&&n<=55215?`ko`:n>=1024&&n<=1279?`ru`:n>=1536&&n<=1791?`ar`:n>=3584&&n<=3711?`th`:n===228||n===246||n===252||n===196||n===214||n===220||n===223?`de`:n>=65&&n<=90||n>=97&&n<=122||n>=192&&n<=591?`en`:``}function analyzeText(n){for(var s=0,c=0,l=0,u=0,d=0,f=0,p=0,m=!1,h=0,g=0;g<n.length;g++){var _=n.charCodeAt(g),v=_>=12352&&_<=12447,y=_>=12448&&_<=12543;v?(m||=(h++,!0),s++):(m=!1,y&&s++),_>=19968&&_<=40959||_>=13312&&_<=19903||_>=63744&&_<=64255?c++:_>=44032&&_<=55215?l++:_>=1024&&_<=1279?u++:_>=1536&&_<=1791?d++:_>=3584&&_<=3711?f++:(_>=65&&_<=90||_>=97&&_<=122||_>=192&&_<=591)&&p++}for(var b=h>=2,x=[[`ja`,s+(b?c:0)],[`zh`,b?0:c],[`ko`,l],[`ru`,u],[`ar`,d],[`th`,f],[`en`,p]],S=[``,0],C=0,w=x;C<w.length;C++){var T=w[C];T[1]>S[1]&&(S=T)}return{primary:S[1]>0?S[0]:void 0,hanIsJa:b}}function splitByScript(n,s){s===void 0&&(s={});for(var c=s.hanIsJa?`ja`:`zh`,l=[],u=null,d=0;d<n.length;d++){var f=classifyChar(n.charCodeAt(d),c),p=n[d];u?f===``||f===u.lang?u.text+=p:u.lang===``?(u.text+=p,u.lang=f):(l.push(u),u={text:p,lang:f}):u={text:p,lang:f}}return u&&l.push(u),l}function preProcessByScript(n,s,c){for(var l,u,d,f=analyzeText(n),p=f.hanIsJa||((l=c?.toLowerCase().startsWith(`ja`))??!1),m=c??f.primary,h=splitByScript(n,{hanIsJa:p}),g=``,_=0,v=h;_<v.length;_++){var y=v[_],b=y.lang||m,x=s.findBestProcessor(y.text,b);g+=(d=(u=x?.preProcess)?.call(x,y.text))??y.text}return g}function useProcessor(n){languageRegistry.register(n)}var blocks=resolveJson(BLOCK_MAP);function anyAscii(n){for(var s,c,l=``,u=0,d=n;u<d.length;u++){var f=d[u],p=f.codePointAt(0);if(p<=127){l+=f;continue}var m=p>>>8,h=p&255,g=(s=blocks[m])??``;l+=(c=g[h])??``}return l}var TOKEN_REGEX=/([\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]+|\w+[''\u2018\u2019]?\w*|[^\w\s\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff])/g,WORD_CHAR_RE=/[\w\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/;function isPunctuationToken(n){return n.length>0&&!WORD_CHAR_RE.test(n)}var Tokenizer=function(){function n(n){n===void 0&&(n={});var s,c,l,u,d,f,p;this.options={stripStress:(s=n.stripStress)??!1,format:(c=n.format)??`ipa`,separator:(l=n.separator)??` `,anyAscii:(u=n.anyAscii)??!1,toneFormat:(d=n.toneFormat)??`unicode`,affricates:(f=n.affricates)??`separate`,language:n.language},this.registry=(p=n.registry)??languageRegistry}return n.prototype._preprocess=function(n){var s,c,l=analyzeText(n),u=l.hanIsJa||((c=(s=this.options.language)?.toLowerCase().startsWith(`ja`))??!1),d=preProcessByScript(n,this.registry,this.options.language),f=this._detectLanguagesAndSegment(d,u),p=f.words,m=f.languageMap,h=f.segments;if(!this.options.anyAscii)return{text:d,languageMap:m,segments:h,primary:l.primary,hanIsJa:u};var g=this._applyAnyAscii(p,m);return{text:g,languageMap:m,segments:h,primary:l.primary,hanIsJa:l.hanIsJa}},n.prototype._detectLanguagesAndSegment=function(n,c){c===void 0&&(c=!1);for(var l=n.split(/(\s+)/),u={},d=[],f=``,p=``,m=0,h=0;h<n.length;h++){var g=n[h],_=this._detectCharLanguage(g);c&&_===`zh`&&(_=`ja`),_===p?f+=g:(f.trim()&&d.push({text:f,language:p||`en`,startIndex:m}),f=g,p=_,m=h)}f.trim()&&d.push({text:f,language:p||`en`,startIndex:m});for(var v=0,y=l;v<y.length;v++){var b=y[v],x=b.trim();if(x&&!PUNCTUATION.includes(x)){var S=detectLanguage(x);c&&S===`zh`&&(S=`ja`),S&&(u[x.toLowerCase()]=S)}}return{words:l,languageMap:u,segments:d}},n.prototype._applyAnyAscii=function(n,c){for(var l=``,u=0,d=n;u<d.length;u++){var f=d[u],p=f.trim();if(p&&!PUNCTUATION.includes(p)){var m=c[p.toLowerCase()];if(m===`zh`)l+=f;else{var h=anyAscii(p);l+=f.replace(p,h),c[h.toLowerCase()]=m}}else l+=f}return l},n.prototype._detectCharLanguage=function(n){var s=n.charCodeAt(0);return s>=19968&&s<=40959||s>=13312&&s<=19903||s>=131072&&s<=173791?`zh`:s>=12352&&s<=12447||s>=12448&&s<=12543?`ja`:s>=44032&&s<=55215||s>=4352&&s<=4607||s>=12592&&s<=12687?`ko`:s>=3584&&s<=3711?`th`:s>=1536&&s<=1791||s>=1872&&s<=1919||s>=64336&&s<=65023||s>=65136&&s<=65279?`ar`:s>=1024&&s<=1279?`ru`:`en`},n.prototype._postProcess=function(n){if(this.options.format===`arpabet`)n=ipaToArpabet(n),this.options.stripStress&&(n=n.replace(/[012]/g,``));else if(this.options.format===`zhuyin`)return n;else this.options.toneFormat===`arrow`&&(n=convertChineseTonesToArrows(n)),this.options.affricates===`ligature`?n=n.replace(/dʒ/g,`ʤ`).replace(/tʃ/g,`ʧ`):this.options.affricates===`tie-bar`&&(n=n.replace(/dʒ/g,`d͡ʒ`).replace(/tʃ/g,`t͡ʃ`)),this.options.stripStress&&(n=n.replace(/[ˈˌ]/g,``));return n},n.prototype._predict=function(n,s,c){var l=this.registry.predictPhonemes(n,s,c);return this._postProcess(l||n)},n.prototype._processTokens=function(n,s){var c=this,l,u;if(s===void 0&&(s=!1),!n?.trim())return[];var d=this._preprocess(n),f=d.text,p=d.languageMap,m=d.primary,h=d.hanIsJa,g=[];if(s)for(var _=void 0;(_=TOKEN_REGEX.exec(f))!==null;){var v=_[1];/^\s+$/.test(v)||v.trim()&&g.push({token:v.trim(),position:_.index})}else{var y=this._smartTokenize(f);g.push.apply(g,y.map(function(n){return{token:n}}))}for(var b=g.filter(function(n){var s=n.token;return!isPunctuationToken(s)}),x=function(n){var s,l,u,d=(u=(l=(s=p[n.toLowerCase()])??detectLanguage(n))??c.options.language)??m;return h&&d===`zh`&&(d=`ja`),d},S=b.length>1,w=b.map(function(n,s){var l,u=x(n.token);if(!S)return{lang:u,pos:void 0};var d=c.registry.findBestProcessor(n.token,u),f=s>0?b[s-1].token:void 0,p=s+1<b.length?b[s+1].token:void 0,m=(l=d?.tagWord)?.call(d,n.token,{prev:f,next:p});return{lang:u,pos:m?.pos}}),T=[],E=0,D=0,O=g;D<O.length;D++){var k=O[D],v=k.token,A=k.position,j=v.trim();if(isPunctuationToken(j)){var M=s&&A!==void 0?{phoneme:j,word:j,position:A}:{phoneme:j};T.push(M);continue}var N=w[E++],P=N.lang,F=N.pos,I=void 0;if(this.options.format===`zhuyin`&&P===`zh`){var L=this.registry.findBestProcessor(j,P);I=(u=(l=L?.textToZhuyin)?.call(L,j))??this._predict(j,P,F)}else I=this._predict(j,P,F);this.options.separator!==` `&&(I=I.split(` `).join(this.options.separator));var R=s&&A!==void 0?{phoneme:I,word:j,position:A}:{phoneme:I};T.push(R)}return T},n.prototype.tokenize=function(n){var s=this._processTokens(n,!1);return s.map(function(n){return n.phoneme})},n.prototype._smartTokenize=function(n){for(var c=[],l;(l=TOKEN_REGEX.exec(n))!==null;){var u=l[1];if(!/^\s+$/.test(u)){if(u.length===1&&PUNCTUATION.includes(u)){c.push(u);continue}u.trim()&&c.push(u.trim())}}return c},n.prototype.tokenizeToString=function(n){for(var c=this.tokenize(n),l=[],u=0;u<c.length;u++){var d=c[u];PUNCTUATION.includes(d)&&l.length>0?l[l.length-1]+=d:l.push(d)}return l.join(this.options.separator)},n.prototype.tokenizeToTokens=function(n){var s=this._processTokens(n,!0);return s.filter(function(n){return`word`in n})},n}();function normalizeArg(n){return n===void 0?{}:n===!0?{returnArray:!0}:typeof n==`string`?{language:n}:n}function phonemize(n,s){s===void 0&&(s={});var c=normalizeArg(s),l=new Tokenizer(c);return c.returnArray?l.tokenizeToTokens(n):l.tokenizeToString(n)}function toIPA(n,s){var c=typeof s==`string`?{language:s}:s??{},l=__assign(__assign({},c),{format:`ipa`});return new Tokenizer(l).tokenizeToString(n)}function toARPABET(n,s){var c=typeof s==`string`?{language:s}:s??{},l=__assign(__assign({},c),{format:`arpabet`});return new Tokenizer(l).tokenizeToString(n)}function toZhuyin(n,s){var c=typeof s==`string`?{language:s}:s??{},l=__assign(__assign({},c),{format:`zhuyin`});return new Tokenizer(l).tokenizeToString(n)}function addPronunciation(n,s,c){if(!n?.trim()||!s?.trim())throw Error(`Both word and pronunciation must be non-empty strings`);var l=languageRegistry.findBestProcessor(n,c);l?.addPronunciation(n.toLowerCase(),s)}function createTokenizer(n){return n===void 0&&(n={}),new Tokenizer(n)}var Phonemizer=function(){function n(n){if(n===void 0&&(n={}),this.registry=new LanguageRegistry,n.processors)for(var s=0,c=n.processors;s<c.length;s++){var l=c[s];this.registry.register(l)}this.defaultLanguage=n.language}return n.prototype.useProcessor=function(n){return this.registry.register(n),this},n.prototype.unregister=function(n){return this.registry.unregister(n)},n.prototype._resolve=function(n){var s,c=normalizeArg(n);return __assign(__assign({},c),{registry:this.registry,language:(s=c.language)??this.defaultLanguage})},n.prototype.phonemize=function(n,s){s===void 0&&(s={});var c=this._resolve(s),l=new Tokenizer(c);return c.returnArray?l.tokenizeToTokens(n):l.tokenizeToString(n)},n.prototype.toIPA=function(n,s){var c,l=typeof s==`string`?{language:s}:s??{};return new Tokenizer(__assign(__assign({},l),{format:`ipa`,registry:this.registry,language:(c=l.language)??this.defaultLanguage})).tokenizeToString(n)},n.prototype.toARPABET=function(n,s){var c,l=typeof s==`string`?{language:s}:s??{};return new Tokenizer(__assign(__assign({},l),{format:`arpabet`,registry:this.registry,language:(c=l.language)??this.defaultLanguage})).tokenizeToString(n)},n.prototype.toZhuyin=function(n,s){var c,l=typeof s==`string`?{language:s}:s??{};return new Tokenizer(__assign(__assign({},l),{format:`zhuyin`,registry:this.registry,language:(c=l.language)??this.defaultLanguage})).tokenizeToString(n)},n.prototype.addPronunciation=function(n,s,c){if(!n?.trim()||!s?.trim())throw Error(`Both word and pronunciation must be non-empty strings`);var l=this.registry.findBestProcessor(n,c??this.defaultLanguage);l?.addPronunciation(n.toLowerCase(),s)},n.prototype.createTokenizer=function(n){var s;return n===void 0&&(n={}),new Tokenizer(__assign(__assign({},n),{registry:this.registry,language:(s=n.language)??this.defaultLanguage}))},n}();function createPhonemizer(n){return n===void 0&&(n={}),new Phonemizer(n)}var phonemizer={phonemize,toIPA,toARPABET,toZhuyin,addPronunciation,createTokenizer,useProcessor,createPhonemizer,Tokenizer,Phonemizer,LanguageRegistry};export{LanguageRegistry,Phonemizer,Tokenizer,addPronunciation,createPhonemizer,createTokenizer,phonemizer as default,phonemize,toARPABET,toIPA,toZhuyin,useProcessor};