phonemize/dist/core.cjs

Fast phonemizer with rule-based G2P prediction. Pure JavaScript implementation.

"use strict";Object.defineProperty(exports,`__esModule`,{value:!0});var utils=require(`./utils-Dq_eIfdp.cjs`),BLOCK_MAP=require(`./anyascii.json`);function _interopNamespaceDefault(t){var a=Object.create(null);return t&&Object.keys(t).forEach(function(o){if(o!==`default`){var s=Object.getOwnPropertyDescriptor(t,o);Object.defineProperty(a,o,s.get?s:{enumerable:!0,get:function(){return t[o]}})}}),a.default=t,Object.freeze(a)}var BLOCK_MAP__namespace=_interopNamespaceDefault(BLOCK_MAP);function normalizeTag(t){return t.toLowerCase()}function tagCovers(t,a){var o=normalizeTag(t),s=normalizeTag(a);return o===s?!0:s.startsWith(o+`-`)}var LanguageRegistry=function(){function t(){this.processors=new Map,this.order=[]}return t.prototype.register=function(t){var a=this.processors.get(t.id);if(a){var o=this.order.indexOf(a);o===-1?this.order.push(t):this.order[o]=t}else this.order.push(t);this.processors.set(t.id,t)},t.prototype.unregister=function(t){var a=this.processors.get(t);return a?(this.processors.delete(t),this.order=this.order.filter(function(a){return a.id!==t}),!0):!1},t.prototype.getProcessor=function(t){return this.processors.get(t)},t.prototype.getProcessorsForLanguage=function(t){for(var a=normalizeTag(t),o=[],s=[],u=0,d=0,f=this.order;d<f.length;d++){for(var p=f[d],m=!1,h=0,g=0,_=p.supportedLanguages;g<_.length;g++){var v=_[g],y=normalizeTag(v);if(y===a){m=!0;break}tagCovers(v,t)&&y.length>h&&(h=y.length)}m?o.push(p):h>0&&s.push({proc:p,matchLen:h,order:u}),u++}return s.sort(function(t,a){return a.matchLen-t.matchLen||t.order-a.order}),o.concat(s.map(function(t){return t.proc}))},t.prototype.getAllProcessors=function(){return this.order.slice()},t.prototype.findBestProcessor=function(t,a){var o;if(a){var s=this.getProcessorsForLanguage(a);if(s.length>0)return s[0]}return(o=this.order[0])??null},t.prototype.predictPhonemes=function(t,a,o){var s=a;if(!s){var c=detectLanguage(t);c&&(s=c)}var l=this.findBestProcessor(t,s);if(!l)return null;var u=l.predict(t,s,o);return u&&u.trim()?u:null},t.prototype.clear=function(){this.processors.clear(),this.order=[]},t.prototype.getSupportedLanguages=function(){for(var t=new Set,a=0,o=this.order;a<o.length;a++)for(var s=o[a],c=0,l=s.supportedLanguages;c<l.length;c++){var u=l[c];t.add(u)}return Array.from(t)},t}(),languageRegistry=new LanguageRegistry,CHINESE_CHARS=/[一-龥]/,JAPANESE_CHARS=/[぀-ヿ]/,KOREAN_CHARS=/[가-힯]/,RUSSIAN_CHARS=/[Ѐ-ӿ]/,GERMAN_CHARS=/[äöüÄÖÜß]/,ARABIC_CHARS=/[؀-ۿ]/,THAI_CHARS=/[฀-๿]/;function detectLanguage(t){return CHINESE_CHARS.test(t)?`zh`:JAPANESE_CHARS.test(t)?`ja`:KOREAN_CHARS.test(t)?`ko`:RUSSIAN_CHARS.test(t)?`ru`:GERMAN_CHARS.test(t)?`de`:ARABIC_CHARS.test(t)?`ar`:THAI_CHARS.test(t)?`th`:null}function classifyChar(t,a){return a===void 0&&(a=`zh`),t>=19968&&t<=40959||t>=13312&&t<=19903||t>=63744&&t<=64255?a:t>=12352&&t<=12447||t>=12448&&t<=12543?`ja`:t>=44032&&t<=55215?`ko`:t>=1024&&t<=1279?`ru`:t>=1536&&t<=1791?`ar`:t>=3584&&t<=3711?`th`:t===228||t===246||t===252||t===196||t===214||t===220||t===223?`de`:t>=65&&t<=90||t>=97&&t<=122||t>=192&&t<=591?`en`:``}function analyzeText(t){for(var a=0,o=0,s=0,c=0,l=0,u=0,d=0,f=!1,p=0,m=0;m<t.length;m++){var h=t.charCodeAt(m),g=h>=12352&&h<=12447,_=h>=12448&&h<=12543;g?(f||=(p++,!0),a++):(f=!1,_&&a++),h>=19968&&h<=40959||h>=13312&&h<=19903||h>=63744&&h<=64255?o++:h>=44032&&h<=55215?s++:h>=1024&&h<=1279?c++:h>=1536&&h<=1791?l++:h>=3584&&h<=3711?u++:(h>=65&&h<=90||h>=97&&h<=122||h>=192&&h<=591)&&d++}for(var v=p>=2,y=[[`ja`,a+(v?o:0)],[`zh`,v?0:o],[`ko`,s],[`ru`,c],[`ar`,l],[`th`,u],[`en`,d]],b=[``,0],x=0,S=y;x<S.length;x++){var C=S[x];C[1]>b[1]&&(b=C)}return{primary:b[1]>0?b[0]:void 0,hanIsJa:v}}function splitByScript(t,a){a===void 0&&(a={});for(var o=a.hanIsJa?`ja`:`zh`,s=[],c=null,l=0;l<t.length;l++){var u=classifyChar(t.charCodeAt(l),o),d=t[l];c?u===``||u===c.lang?c.text+=d:c.lang===``?(c.text+=d,c.lang=u):(s.push(c),c={text:d,lang:u}):c={text:d,lang:u}}return c&&s.push(c),s}function preProcessByScript(t,a,o){for(var s,c,l,u=analyzeText(t),d=u.hanIsJa||((s=o?.toLowerCase().startsWith(`ja`))??!1),f=o??u.primary,p=splitByScript(t,{hanIsJa:d}),m=``,h=0,g=p;h<g.length;h++){var _=g[h],v=_.lang||f,y=a.findBestProcessor(_.text,v);m+=(l=(c=y?.preProcess)?.call(y,_.text))??_.text}return m}function useProcessor(t){languageRegistry.register(t)}var blocks=utils.resolveJson(BLOCK_MAP__namespace);function anyAscii(t){for(var a,o,s=``,c=0,l=t;c<l.length;c++){var u=l[c],d=u.codePointAt(0);if(d<=127){s+=u;continue}var f=d>>>8,p=d&255,m=(a=blocks[f])??``;s+=(o=m[p])??``}return s}var TOKEN_REGEX=/([\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]+|\w+[''\u2018\u2019]?\w*|[^\w\s\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff])/g,WORD_CHAR_RE=/[\w\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/;function isPunctuationToken(t){return t.length>0&&!WORD_CHAR_RE.test(t)}var Tokenizer=function(){function a(t){t===void 0&&(t={});var a,o,s,c,l,u,f;this.options={stripStress:(a=t.stripStress)??!1,format:(o=t.format)??`ipa`,separator:(s=t.separator)??` `,anyAscii:(c=t.anyAscii)??!1,toneFormat:(l=t.toneFormat)??`unicode`,affricates:(u=t.affricates)??`separate`,language:t.language},this.registry=(f=t.registry)??languageRegistry}return a.prototype._preprocess=function(t){var a,o,s=analyzeText(t),c=s.hanIsJa||((o=(a=this.options.language)?.toLowerCase().startsWith(`ja`))??!1),l=preProcessByScript(t,this.registry,this.options.language),u=this._detectLanguagesAndSegment(l,c),d=u.words,f=u.languageMap,p=u.segments;if(!this.options.anyAscii)return{text:l,languageMap:f,segments:p,primary:s.primary,hanIsJa:c};var m=this._applyAnyAscii(d,f);return{text:m,languageMap:f,segments:p,primary:s.primary,hanIsJa:s.hanIsJa}},a.prototype._detectLanguagesAndSegment=function(a,o){o===void 0&&(o=!1);for(var s=a.split(/(\s+)/),c={},l=[],u=``,d=``,f=0,p=0;p<a.length;p++){var m=a[p],h=this._detectCharLanguage(m);o&&h===`zh`&&(h=`ja`),h===d?u+=m:(u.trim()&&l.push({text:u,language:d||`en`,startIndex:f}),u=m,d=h,f=p)}u.trim()&&l.push({text:u,language:d||`en`,startIndex:f});for(var g=0,_=s;g<_.length;g++){var v=_[g],b=v.trim();if(b&&!utils.PUNCTUATION.includes(b)){var x=detectLanguage(b);o&&x===`zh`&&(x=`ja`),x&&(c[b.toLowerCase()]=x)}}return{words:s,languageMap:c,segments:l}},a.prototype._applyAnyAscii=function(a,o){for(var s=``,c=0,l=a;c<l.length;c++){var u=l[c],d=u.trim();if(d&&!utils.PUNCTUATION.includes(d)){var f=o[d.toLowerCase()];if(f===`zh`)s+=u;else{var p=anyAscii(d);s+=u.replace(d,p),o[p.toLowerCase()]=f}}else s+=u}return s},a.prototype._detectCharLanguage=function(t){var a=t.charCodeAt(0);return a>=19968&&a<=40959||a>=13312&&a<=19903||a>=131072&&a<=173791?`zh`:a>=12352&&a<=12447||a>=12448&&a<=12543?`ja`:a>=44032&&a<=55215||a>=4352&&a<=4607||a>=12592&&a<=12687?`ko`:a>=3584&&a<=3711?`th`:a>=1536&&a<=1791||a>=1872&&a<=1919||a>=64336&&a<=65023||a>=65136&&a<=65279?`ar`:a>=1024&&a<=1279?`ru`:`en`},a.prototype._postProcess=function(a){if(this.options.format===`arpabet`)a=utils.ipaToArpabet(a),this.options.stripStress&&(a=a.replace(/[012]/g,``));else if(this.options.format===`zhuyin`)return a;else this.options.toneFormat===`arrow`&&(a=utils.convertChineseTonesToArrows(a)),this.options.affricates===`ligature`?a=a.replace(/dʒ/g,`ʤ`).replace(/tʃ/g,`ʧ`):this.options.affricates===`tie-bar`&&(a=a.replace(/dʒ/g,`d͡ʒ`).replace(/tʃ/g,`t͡ʃ`)),this.options.stripStress&&(a=a.replace(/[ˈˌ]/g,``));return a},a.prototype._predict=function(t,a,o){var s=this.registry.predictPhonemes(t,a,o);return this._postProcess(s||t)},a.prototype._processTokens=function(t,a){var o=this,s,c;if(a===void 0&&(a=!1),!t?.trim())return[];var l=this._preprocess(t),u=l.text,d=l.languageMap,f=l.primary,p=l.hanIsJa,m=[];if(a)for(var h=void 0;(h=TOKEN_REGEX.exec(u))!==null;){var g=h[1];/^\s+$/.test(g)||g.trim()&&m.push({token:g.trim(),position:h.index})}else{var _=this._smartTokenize(u);m.push.apply(m,_.map(function(t){return{token:t}}))}for(var v=m.filter(function(t){var a=t.token;return!isPunctuationToken(a)}),b=function(t){var a,s,c,l=(c=(s=(a=d[t.toLowerCase()])??detectLanguage(t))??o.options.language)??f;return p&&l===`zh`&&(l=`ja`),l},x=v.length>1,S=v.map(function(t,a){var s,c=b(t.token);if(!x)return{lang:c,pos:void 0};var l=o.registry.findBestProcessor(t.token,c),u=a>0?v[a-1].token:void 0,d=a+1<v.length?v[a+1].token:void 0,f=(s=l?.tagWord)?.call(l,t.token,{prev:u,next:d});return{lang:c,pos:f?.pos}}),C=[],w=0,T=0,E=m;T<E.length;T++){var D=E[T],g=D.token,O=D.position,k=g.trim();if(isPunctuationToken(k)){var A=a&&O!==void 0?{phoneme:k,word:k,position:O}:{phoneme:k};C.push(A);continue}var j=S[w++],M=j.lang,N=j.pos,P=void 0;if(this.options.format===`zhuyin`&&M===`zh`){var F=this.registry.findBestProcessor(k,M);P=(c=(s=F?.textToZhuyin)?.call(F,k))??this._predict(k,M,N)}else P=this._predict(k,M,N);this.options.separator!==` `&&(P=P.split(` `).join(this.options.separator));var I=a&&O!==void 0?{phoneme:P,word:k,position:O}:{phoneme:P};C.push(I)}return C},a.prototype.tokenize=function(t){var a=this._processTokens(t,!1);return a.map(function(t){return t.phoneme})},a.prototype._smartTokenize=function(a){for(var o=[],s;(s=TOKEN_REGEX.exec(a))!==null;){var c=s[1];if(!/^\s+$/.test(c)){if(c.length===1&&utils.PUNCTUATION.includes(c)){o.push(c);continue}c.trim()&&o.push(c.trim())}}return o},a.prototype.tokenizeToString=function(a){for(var o=this.tokenize(a),s=[],c=0;c<o.length;c++){var l=o[c];utils.PUNCTUATION.includes(l)&&s.length>0?s[s.length-1]+=l:s.push(l)}return s.join(this.options.separator)},a.prototype.tokenizeToTokens=function(t){var a=this._processTokens(t,!0);return a.filter(function(t){return`word`in t})},a}();function normalizeArg(t){return t===void 0?{}:t===!0?{returnArray:!0}:typeof t==`string`?{language:t}:t}function phonemize(t,a){a===void 0&&(a={});var o=normalizeArg(a),s=new Tokenizer(o);return o.returnArray?s.tokenizeToTokens(t):s.tokenizeToString(t)}function toIPA(a,o){var s=typeof o==`string`?{language:o}:o??{},c=utils.__assign(utils.__assign({},s),{format:`ipa`});return new Tokenizer(c).tokenizeToString(a)}function toARPABET(a,o){var s=typeof o==`string`?{language:o}:o??{},c=utils.__assign(utils.__assign({},s),{format:`arpabet`});return new Tokenizer(c).tokenizeToString(a)}function toZhuyin(a,o){var s=typeof o==`string`?{language:o}:o??{},c=utils.__assign(utils.__assign({},s),{format:`zhuyin`});return new Tokenizer(c).tokenizeToString(a)}function addPronunciation(t,a,o){if(!t?.trim()||!a?.trim())throw Error(`Both word and pronunciation must be non-empty strings`);var s=languageRegistry.findBestProcessor(t,o);s?.addPronunciation(t.toLowerCase(),a)}function createTokenizer(t){return t===void 0&&(t={}),new Tokenizer(t)}var Phonemizer=function(){function a(t){if(t===void 0&&(t={}),this.registry=new LanguageRegistry,t.processors)for(var a=0,o=t.processors;a<o.length;a++){var s=o[a];this.registry.register(s)}this.defaultLanguage=t.language}return a.prototype.useProcessor=function(t){return this.registry.register(t),this},a.prototype.unregister=function(t){return this.registry.unregister(t)},a.prototype._resolve=function(a){var o,s=normalizeArg(a);return utils.__assign(utils.__assign({},s),{registry:this.registry,language:(o=s.language)??this.defaultLanguage})},a.prototype.phonemize=function(t,a){a===void 0&&(a={});var o=this._resolve(a),s=new Tokenizer(o);return o.returnArray?s.tokenizeToTokens(t):s.tokenizeToString(t)},a.prototype.toIPA=function(a,o){var s,c=typeof o==`string`?{language:o}:o??{};return new Tokenizer(utils.__assign(utils.__assign({},c),{format:`ipa`,registry:this.registry,language:(s=c.language)??this.defaultLanguage})).tokenizeToString(a)},a.prototype.toARPABET=function(a,o){var s,c=typeof o==`string`?{language:o}:o??{};return new Tokenizer(utils.__assign(utils.__assign({},c),{format:`arpabet`,registry:this.registry,language:(s=c.language)??this.defaultLanguage})).tokenizeToString(a)},a.prototype.toZhuyin=function(a,o){var s,c=typeof o==`string`?{language:o}:o??{};return new Tokenizer(utils.__assign(utils.__assign({},c),{format:`zhuyin`,registry:this.registry,language:(s=c.language)??this.defaultLanguage})).tokenizeToString(a)},a.prototype.addPronunciation=function(t,a,o){if(!t?.trim()||!a?.trim())throw Error(`Both word and pronunciation must be non-empty strings`);var s=this.registry.findBestProcessor(t,o??this.defaultLanguage);s?.addPronunciation(t.toLowerCase(),a)},a.prototype.createTokenizer=function(a){var o;return a===void 0&&(a={}),new Tokenizer(utils.__assign(utils.__assign({},a),{registry:this.registry,language:(o=a.language)??this.defaultLanguage}))},a}();function createPhonemizer(t){return t===void 0&&(t={}),new Phonemizer(t)}var phonemizer={phonemize,toIPA,toARPABET,toZhuyin,addPronunciation,createTokenizer,useProcessor,createPhonemizer,Tokenizer,Phonemizer,LanguageRegistry};exports.LanguageRegistry=LanguageRegistry,exports.Phonemizer=Phonemizer,exports.Tokenizer=Tokenizer,exports.addPronunciation=addPronunciation,exports.createPhonemizer=createPhonemizer,exports.createTokenizer=createTokenizer,exports.default=phonemizer,exports.phonemize=phonemize,exports.toARPABET=toARPABET,exports.toIPA=toIPA,exports.toZhuyin=toZhuyin,exports.useProcessor=useProcessor;