i18n-text-tools/dist/index.js

一个用于自动提取、管理和同步项目多语言文本的命令行工具，支持 Vue3、JS/TS 文件的国际化文本收集、唯一key生成、重复检测、与YiCAT平台集成等功能。

import{Command as ue}from"commander";import x from"colors";import w from"path";import J from"fs";import se from"fs";import oe from"path";import we from"colors";import D,{existsSync as G,readFileSync as Z}from"fs";import U from"path";import A from"colors";import ee from"crypto";import te from"figlet";import ne from"writejson";var y=(process.env.NODE_ENV||"").trim()==="dev_debug",_=o=>{let e=U.dirname(o);D.existsSync(e)||(_(e),D.mkdirSync(e))},S=o=>{_(o),D.existsSync(o)||D.writeFileSync(o,"")},$=(o,e)=>{if(!G(o))return e;let t=Z(o).toString("utf-8");return t?JSON.parse(t):e},k=(o,e)=>{ne.sync(o,e,{spaces:2})},K=o=>{console.log(A.yellow(o))};var v=o=>{let e=ee.createHash("md5");return e.update(o),e.digest("hex")},R=o=>{let e=Object.keys(o).sort(),t={};return e.forEach(s=>{t[s]=o[s]}),t},z=async()=>{let o=await te("I 1 8 N - T E X T - T O O L S");console.log(A.green(o))};var L=class{srcCodeDir="src";outputDir="i18n-messages";translateFunctionName=["t","$t"];defaultLanuage="zh";targetLanuages=["en","ja","ko","vi","th","id","ms","pt","ru","es"];i18nLocaleDir="i18n/locales";targetFormat="json";microsoftInfo={endpoint:"https://api.cognitive.microsofttranslator.com",key:"",location:"eastasia"};xiaoniuTrans;constructor(){let e=oe.join(process.cwd(),"package.json");if(!se.existsSync(e))console.warn("Cannot find package.json file. It's recommended to run this command in your project root.");else try{let s=$(e,{}).i18nPickConfig;if(!s)throw new Error("Cannot find i18nPickConfig field in package.json.");s.srcCodeDir&&(this.srcCodeDir=s.srcCodeDir),s.outputDir&&(this.outputDir=s.outputDir),s.translateFunctionName&&(this.translateFunctionName=s.translateFunctionName),s.defaultLanuage&&(this.defaultLanuage=s.defaultLanuage),s.targetLanuages&&s.targetLanuages.length&&(this.targetLanuages=s.targetLanuages),s.i18nLocaleDir&&(this.i18nLocaleDir=s.i18nLocaleDir),s.targetFormat&&(this.targetFormat=s.targetFormat),s.microsoftInfo&&(this.microsoftInfo=s.microsoftInfo),s.xiaoniuTrans&&(this.xiaoniuTrans=s.xiaoniuTrans)}catch(t){console.warn(t.message)}}};import{glob as M}from"glob";import*as H from"vue/compiler-sfc";import q from"@babel/parser";import re from"@babel/traverse";import W from"@babel/generator";import ie from"dayjs";var Q=async(o,e,t="zh-Hans",s="en")=>{if(!e||!e.endpoint||!e.key||!e.location)return"";y&&console.log("try translate:",o,t,s);let r=await(await fetch(`${e.endpoint}/translate?api-version=3.0&from=${t}&to=${s}`,{method:"POST",headers:{"Ocp-Apim-Subscription-Key":e.key,"Ocp-Apim-Subscription-Region":e.location,"Content-type":"application/json","X-ClientTraceId":Date.now().toString()},body:JSON.stringify([{text:o}])})).json();try{let n=r[0].translations.find(a=>a.to.includes(s)).text;return console.log("\u5FAE\u8F6F\u7FFB\u8BD1\u7ED3\u679C:",n),n}catch(n){throw console.log("\u5FAE\u8F6F\u7FFB\u8BD1\u9519\u8BEF"),console.log(JSON.stringify(r)),n}},V=async(o,e,t="zh",s="en")=>{if(!e||!e.appId||!e.apiKey)return"";y&&console.log("try translate:",o,t,s);let i=Math.floor(Date.now()/1e3),r=[`apikey=${e.apiKey}`,`appId=${e.appId}`,`from=${t}`,`to=${s}`,`srcText=${o}`,`timestamp=${i}`].sort((c,l)=>c.localeCompare(l)).join("&"),n=v(r),d=(await(await fetch("https://api.niutrans.com/v2/text/translate",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({from:t,to:s,appId:e.appId,srcText:o,timestamp:i,authStr:n})})).json()).tgtText;return console.log("\u5C0F\u725B\u7FFB\u8BD1\u7ED3\u679C:",d),d};var Y=o=>o&&o.type==="StringLiteral",ae=o=>o&&o.type==="TemplateLiteral",le=o=>o&&o.type==="ObjectExpression",ce=/^\d{14}_[0-9a-zA-Z]{6}/,C=()=>`${ie().format("YYYYMMDDHHmmss")}_${Math.random().toString(16).slice(4,10)}`,b=o=>{let e="",t=o,s=o.match(ce);return s&&(e=s[0],t=o.slice(e.length+1)),{msgId:e,originalTranlationText:t}},F=o=>/[\u4e00-\u9fa5]/.test(o),N=(o,e,t)=>{t(o,e),o.content&&o.content.type&&t(o.content,o),o.exp&&o.exp.type&&t(o.exp,o),o.value&&o.value.type&&t(o.value,o),o.props&&o.props.length&&o.props.forEach(s=>{N(s,o,t)}),o.children&&o.children.length&&o.children.forEach(s=>{N(s,o,t)}),o.properties&&o.properties.length&&o.properties.forEach(s=>{N(s,o,t)})},P=class{config;fileItems=[];changedFiles=[];constructor(){this.config=new L}async start(e){await z(),y&&K("-------------------------\u5F00\u53D1\u73AF\u5883\u8FD0\u884C\u4E2D------------------------"),console.log(x.green("start:"),"start extract files in dir: "+this.config.srcCodeDir+"..."),this.fileItems=this.scanFiles(),this.changedFiles=this.fileItems.filter(t=>t.isFileNeedExtract),console.log(),console.log(`-------------------------- ${x.green(`${this.changedFiles.length}`)}  files changed------------------`),console.log(),this.processFiles(),await this.saveTranslationData(),this.changedFiles.forEach(t=>{t.newFileContent!==t.fileContent&&!y&&(t.md5=v(t.newFileContent),J.writeFileSync(t.fileName,t.newFileContent))}),this.saveFileCacheInfo(),this.generateResultFile()}scanFiles(){let e=this.getFileCacheInfo(),t=M.sync(this.config.srcCodeDir+"/**/*.vue"),s=M.sync(this.config.srcCodeDir+"/**/*.ts"),i=M.sync(this.config.srcCodeDir+"/**/*.js"),r=t.concat(s).concat(i);return r.sort(),r.map(a=>{let g=J.readFileSync(a,"utf-8"),d=v(g),c=e[a]!==d||y;return{fileId:"",fileName:a,fileComment:"",fileContent:g,newFileContent:c?g:"",md5:d,translations:[],isFileNeedExtract:c}})}async processFiles(){for(let e=0;e<this.changedFiles.length;e++){let t=this.changedFiles[e];if(console.log(x.green("process: ------------------"),x.white(`${t.fileName}`),x.green(" ------------------")),!!t.fileContent&&(t.fileName.endsWith(".ts")||t.fileName.endsWith(".js")?this.processTsFile(t):t.fileName.endsWith(".vue")?this.processVueFile(t):t.fileName.endsWith(".html"),t.newFileContent!==t.fileContent&&console.log(x.yellow("file content transformed: "),t.fileName),(t.newFileContent!==t.fileContent||y)&&(t.md5=v(t.fileContent),y))){let s=w.join(this.config.outputDir,t.fileName);_(s),J.writeFileSync(s,t.newFileContent)}}}getCodeAst(e){try{return q.parse(e,{attachComment:!1,sourceType:"module",plugins:["typescript"]})}catch(t){throw t}}getAstCommentByKey(e,t){if(!e||!e.comments||!e.comments.length)return"";let s=e.comments.find(i=>i.value.includes(t));return s?s.value.replace(t,"").trim():""}dealJSON(e){e.ast=q.parseExpression(e.sourceCode,{attachComment:!1,sourceType:"module",plugins:["typescript"]});let t={fileId:"",fileComment:"",newContent:e.sourceCode,translations:[]},s=0;return N(e.ast,null,(i,r)=>{if(i.type==="ObjectProperty"&&i.value&&i.value.type==="StringLiteral"){let n=i.value,a=n.value;if(F(a)){let{msgId:g,originalTranlationText:d}=b(a);if(!g){g=C();let c=n.extra.raw[0]==="'",l=g+"_"+d,f=c?`$t('${l}')`:`$t("${l}")`;t.newContent=t.newContent.substring(0,n.start+s)+f+t.newContent.substring(n.end+s),s+=f.length-(n.end-n.start),t.translations.push({msgId:g,sourceText:d})}}}if(i.type==="CallExpression"&&this.config.translateFunctionName.includes(i.callee.name)){let n=i.arguments[0],a=n.value;if(F(a)){let{msgId:g,originalTranlationText:d}=b(a);if(!g){g=C();let c=n.extra.raw[0]==="'",l=g+"_"+d,f=c?`'${l}'`:`"${l}"`;t.newContent=t.newContent.substring(0,n.start+s)+f+t.newContent.substring(n.end+s),s+=f.length-(n.end-n.start)}t.translations.push({msgId:g,sourceText:d})}}}),t}dealJsCode(e){if(e.replaceTranslateFunctionName=e.replaceTranslateFunctionName||"t",!e.sourceCode)throw new Error("sourceCode is required");if(!e.fileName)throw new Error("fileName is required");if(!e.ast)try{e.ast=this.getCodeAst(e.sourceCode)}catch{return this.dealJSON(e)}let t={fileId:"",fileComment:"",newContent:e.sourceCode,translations:[]};e.needFileId&&(t.fileId=this.getAstCommentByKey(e.ast,"@i18n-file-id:")),e.needFileComment&&(t.fileComment=this.getAstCommentByKey(e.ast,"@i18n-file-comment:"));let{sourceCode:s}=e,i=0;return y&&console.log("\u5F00\u59CB\u89E3\u6790js\u4EE3\u7801*******************************"),re.default(e.ast,{enter(r){},TemplateLiteral(r){if(r.parent.type==="VariableDeclarator"){let n=r.node.expressions.concat(r.node.quasis);n.sort((d,c)=>d.start-c.start);let a=[],g=n.map(d=>{if(d.type==="TemplateElement")return d.value.raw;if(d.type==="Identifier")return a.push(d.name),"{"+(a.length-1)+"}"}).join("");if(F(g)){let d=C(),c=g,l=`${e.replaceTranslateFunctionName}('${g}', [${a.join(", ")}])`;t.newContent=t.newContent.substring(0,r.node.start+i)+l+t.newContent.substring(r.node.end+i),i+=l.length-(r.node.end-r.node.start),t.translations.push({msgId:d,sourceText:c})}}},StringLiteral(r){let n=r.node.value;if(F(n)){if(r.container.type==="ObjectProperty"&&r.node.value){let{msgId:a,originalTranlationText:g}=b(n);if(!a){a=C();let d=r.node.extra.raw[0]==="'",c=a+"_"+g,l=d?`${e.replaceTranslateFunctionName}('${c}')`:`${e.replaceTranslateFunctionName}("${c}")`;t.newContent=t.newContent.substring(0,r.node.start+i)+l+t.newContent.substring(r.node.end+i),i+=l.length-r.node.extra.raw.length}t.translations.push({msgId:a,sourceText:g})}if(r.container.type==="VariableDeclarator"){let a=C(),g=n,d=r.node.extra.raw[0]==="'",c=a+"_"+g,l=d?`${e.replaceTranslateFunctionName}('${c}')`:`${e.replaceTranslateFunctionName}("${c}")`;t.newContent=t.newContent.substring(0,r.node.start+i)+l+t.newContent.substring(r.node.end+i),i+=l.length-r.node.extra.raw.length,t.translations.push({msgId:a,sourceText:g})}if(r.container.type==="ExpressionStatement"){let a=C(),g=n,d=a+"_"+g,c=`${e.replaceTranslateFunctionName}('${d}')`;t.newContent=t.newContent.substring(0,r.node.start+i)+c+t.newContent.substring(r.node.end+i),i+=c.length-r.node.extra.raw.length,t.translations.push({msgId:a,sourceText:g})}}},CallExpression:r=>{var d,c,l,f;let n=r.node,a=this.config.translateFunctionName.includes(n.callee.name),g=s.substring(n.start,n.end);if(a){let m=n.arguments[0];if(Y(m)){let u=m.value,{msgId:p,originalTranlationText:h}=b(u);if(!p){p=C();let T=m.extra.raw[0]==="'";m.value=p+"_"+h,m.extra.raw=T?`'${m.value}'`:`"${m.value}"`,m.extra.rawValue=m.value;let I=W.default(n,{}).code;t.newContent=t.newContent.substring(0,n.start+i)+I+t.newContent.substring(n.end+i),i+=I.length-g.length}t.translations.push({msgId:p,sourceText:h})}else if(ae(n.arguments[0])){let u=s.substring(n.arguments[0].start,n.arguments[0].end);console.log(x.bgRed("\u4E0D\u652F\u6301\u6A21\u677F\u5B57\u7B26\u4E32: "),x.blue(e.fileName),": ",x.yellow(u),', please use placeholder string instead like: "hello {0}"')}else if(le(n.arguments[0])){let u=n.arguments[0],p=(c=(d=u.properties.find(T=>T.key.name==="t"))==null?void 0:d.value)==null?void 0:c.value,h=((f=(l=u.properties.find(T=>T.key.name==="id"))==null?void 0:l.value)==null?void 0:f.value)||p;t.translations.push({msgId:h,sourceText:p})}else console.log(x.red("no handler for this translation function call: ")),console.log(g)}else{let m=!1;n.arguments.forEach((u,p)=>{var h;if(Y(u)){let T=u.value;if(F(T)){if(((h=n.callee.object)==null?void 0:h.name)==="console")return;let{msgId:j,originalTranlationText:E}=b(T);if(!j){j=C();let X=u.extra.raw[0]==="'";u.value=j+"_"+E,u.extra.raw=X?`'${u.value}'`:`"${u.value}"`,u.extra.rawValue=u.value;let O=`t(${W.default(u,{}).code})`;t.newContent=t.newContent.substring(0,u.start+i)+O+t.newContent.substring(u.end+i),i+=O.length-(u.end-u.start)}m=!0,t.translations.push({msgId:j,sourceText:E})}}})}}}),t}dealVueTemplate(e,t,s,i){let r={newContent:t,translations:[]};return N(e,null,(n,a)=>{var d,c;let g=["MemberExpression"];if((d=n.ast)!=null&&d.type&&g.includes(n.ast.type)){y&&console.log("\u5FFD\u7565AST",n.ast.type);return}if(n.type!==1){if(n.type===2){if(a&&a.type!==1)return;let l=[],f=n.loc.start.offset,m=/((\n|\r|\r\n)\s*){1,}/,u=n.loc.source,p=u.match(m);for(;p;){let h=u.slice(0,p.index);h&&(l.push({text:h,start:f,end:f+h.length}),f+=h.length,u=u.slice(h.length)),p[0]&&(f+=p[0].length,u=u.slice(p[0].length)),p=u.match(m)}u&&l.push({text:u,start:f,end:f+u.length}),l.forEach(h=>{if(F(h.text)){let T=C(),I=h.text,E=`{{ $t('${T+"_"+I}') }}`;i(h.start,h.end,E),r.translations.push({msgId:T,sourceText:I})}})}else if(n.type===4){if(!n.ast)return;if(n.ast.type==="StringLiteral"&&F(n.ast.value)){let l=n.ast.value,{msgId:f,originalTranlationText:m}=b(l);if(!f){f=C();let u=n.content[0]==="'",p=f+"_"+m,h=u?`$t('${p}')`:`$t("${p}")`;i(n.loc.start.offset,n.loc.end.offset,h)}r.translations.push({msgId:f,sourceText:m})}else{let l=this.dealJsCode({ast:null,sourceCode:n.content,fileName:s,needFileId:!1,needFileComment:!1,replaceTranslateFunctionName:"$t"});if(!l)return;(l==null?void 0:l.newContent)!==n.content&&i(n.loc.start.offset,n.loc.end.offset,l.newContent),r.translations.push(...l.translations)}}else if(n.type===6&&F((c=n.value)==null?void 0:c.content)){let l=C(),f=n.value.content,m=l+"_"+f,p=n.value.loc.source[0]==="'"?`:${n.name}='$t("${m}")'`:`:${n.name}="$t('${m}')"`;i(n.loc.start.offset,n.loc.end.offset,p),r.translations.push({msgId:l,sourceText:f})}}}),r}processTsFile(e){let t=e.fileContent,s=this.dealJsCode({ast:null,sourceCode:t,fileName:e.fileName,needFileId:!0,needFileComment:!0,replaceTranslateFunctionName:"t"});s&&(s.fileId||(console.warn(x.yellow("No i18n file id\uFF1A "),"The file "+x.red(`"${e.fileName}"`)+' has no i18n-file-id in line comment, please add it as first line like: "// @i18n-file-id: your-file-id"'),s.fileId=w.basename(e.fileName)),e.fileId=s.fileId,e.translations=s.translations,e.newFileContent=s.newContent)}processVueFile(e){let t=e.fileContent,s=H.parse(t,{filename:e.fileName}),i="",r="",n=0,a=s.descriptor.source,g=[];[s.descriptor.script,s.descriptor.scriptSetup,s.descriptor.template].filter(c=>c).sort((c,l)=>c.loc.start.offset-l.loc.start.offset).forEach(c=>{if(c.type==="script"){let l=this.dealJsCode({ast:null,sourceCode:c.content,fileName:e.fileName,needFileId:!0,needFileComment:!0,replaceTranslateFunctionName:"t"});if(!l)return;a=a.substring(0,c.loc.start.offset+n)+l.newContent+a.substring(c.loc.end.offset+n),n+=l.newContent.length-c.content.length,!i&&l.fileId&&(i=l.fileId),!r&&l.fileComment&&(r=l.fileComment),g.push(...l.translations)}else if(c.type==="template"){let l=c.ast.source;console.log("\u5F00\u59CB\u5904\u7406template\u6587\u672C\u63D0\u53D6");let f=this.dealVueTemplate(c.ast,l,e.fileName,(m,u,p)=>{a=a.substring(0,m+n)+p+a.substring(u+n),n+=p.length-(u-m)});g.push(...f.translations)}}),i||(console.warn(x.yellow("No File Id\uFF1A "),"The file "+x.red(`"${e.fileName}"`)+' has no i18n file id in line comment, please add it as first line like: "// @i18n-file-id: your-file-id"'),i=w.basename(e.fileName)),e.fileId=i,e.fileComment=r,e.translations=g,e.newFileContent=a}getFileCacheInfo(){let e=this.config.outputDir,t=w.join(e,".file_md5.json");return S(t),$(t,JSON.stringify({}))}saveFileCacheInfo(){let e=this.fileItems.reduce((i,r)=>(i[r.fileName]=r.md5,i),{}),t=this.config.outputDir,s=w.join(t,".file_md5.json");S(s),k(s,R(e))}getOldTranslationTextItems(){let e=this.config.outputDir,t=w.join(e,".translation_items.json");return S(t),$(t,[])}async saveAllTranslationTextItems(e){let t=this.config.outputDir,s=w.join(t,".translation_items.json"),i=new Map;if(e.forEach(r=>{i.set(r.sourceText,r.targetText)}),!y)for(let r=0;r<e.length;r++){let n=e[r];try{if(!n.targetText){let a=i.get(n.sourceText);a&&(n.targetText=a)}n.targetText||(n.targetText=await V(n.sourceText,this.config.xiaoniuTrans)),n.targetText||(n.targetText=await Q(n.sourceText,this.config.microsoftInfo))}catch{}}S(s),k(s,e)}generateResultFile(){let e=this.getOldTranslationTextItems(),t={};e.forEach(n=>{n.isExistInFile&&(t[n.msgId+"_"+n.sourceText]=n.sourceText)});let s=this.config.outputDir,i=w.join(s,".translation_map.json");S(i);let r=R(t);k(i,r)}async saveTranslationData(){let e=this.getOldTranslationTextItems(),t=e.reduce((r,n)=>(r[n.msgId]=n,r),{}),s=[],i={};this.changedFiles.forEach(r=>{r.translations.forEach(n=>{if(i[n.msgId]=i[n.msgId]||0,i[n.msgId]++,i[n.msgId]>1&&console.log(x.bgRed("Repeat Text"),r.fileName,n,"\u8BF7\u4FEE\u6539msgId"),t[n.msgId]){let a=t[n.msgId];a.sourceText!==n.sourceText&&(a.targetText=""),a.fileComment=r.fileComment,a.sourceText=n.sourceText,a.msgId=n.msgId,a.isExistInFile=!0}else s.push({fileId:r.fileId,filePath:"",fileComment:r.fileComment,sourceText:n.sourceText,msgId:n.msgId,isExistInFile:!0,targetText:""})})}),await this.saveAllTranslationTextItems(e.concat(s))}};var B=class{run(){let e=new ue;e.name("i18n-text-tools").version("1.0.0"),e.command("extract").option("-a, --all","extract all files").description("scan the src dir and extract translations").action(()=>{new P().start()}),e.command("upload").description("upload extracted files to yicat").action(()=>{}),e.command("apply").description("apply latest messages to i18n locales dir").action(()=>{}),e.command("fetch").description("fetch translations from yicat").action(()=>{}),e.parse()}};export{B as CustomCommand};