@mike-lischke/antlr-tgen/dist/index.js

A generator for antlr-ng runtime test cases

#!/usr/bin/env node
import ce from"@readme/better-ajv-errors";import{Ajv as le}from"ajv";import q from"chalk";import{program as Q}from"commander";import{existsSync as X,readFileSync as Y}from"fs";import{globSync as ge}from"glob";import{dirname as M,join as A}from"path";import{fileURLToPath as _}from"url";import*as D from"path";import*as p from"fs";var L=class o{static writeFile(e,t,n){try{p.writeFileSync(D.join(e,t),n,{encoding:"utf-8"})}catch(r){if(r instanceof Error)console.error("can't write file"),console.error(r.stack);else throw r}}static readFile(e,t){try{return p.readFileSync(e+"/"+t,{encoding:"utf-8"})}catch(n){if(n instanceof Error)console.error("can't read file"),console.error(n.stack);else throw n}return null}static mkdir(e){p.mkdirSync(e,{recursive:!0})}static deleteDirectory(e){let t=p.statSync(e);t.isDirectory()&&!t.isSymbolicLink()?p.rmdirSync(e,{recursive:!0}):p.unlinkSync(e)}static isLink(e){try{return p.lstatSync(e).isSymbolicLink()}catch{return!1}}static copyFileOrFolder(e,t){p.statSync(e).isDirectory()?(p.mkdirSync(t,{recursive:!0}),p.readdirSync(e).forEach(r=>{o.copyFileOrFolder(D.join(e,r),D.join(t,r))})):(p.statSync(t).isDirectory()&&(t=D.join(t,D.basename(e))),p.copyFileSync(e,t))}};import{mkdirSync as te,readFileSync as J,readdirSync as K,statSync as re,writeFileSync as j}from"fs";import H from"node:readline";import{join as f}from"path";import{Tool as se}from"antlr-ng";import ne from"chalk";import{ST as $,STGroup as B,STGroupFile as ie,StringRenderer as oe}from"stringtemplate4ts";var N=(r=>(r[r.Lexer=0]="Lexer",r[r.Parser=1]="Parser",r[r.CompositeLexer=2]="CompositeLexer",r[r.CompositeParser=3]="CompositeParser",r))(N||{});var C=class o{static descriptors=new Map;static path;static{o.descriptors.set("LexerExec",[o.getLineSeparatorLfDescriptor(),o.getLineSeparatorCrLfDescriptor(),o.getLargeLexerDescriptor(),o.getAtnStatesSizeMoreThan65535Descriptor()]),o.descriptors.set("ParserExec",[o.getMultiTokenAlternativeDescriptor()])}static getLineSeparatorLfDescriptor(){return{testType:0,name:"LineSeparatorLf",notes:"",input:`1
2
3`,output:`[@0,0:0='1',<1>,1:0]
[@1,1:1='\\n',<2>,1:1]
[@2,2:2='2',<1>,2:0]
[@3,3:3='\\n',<2>,2:1]
[@4,4:4='3',<1>,3:0]
[@5,5:4='<EOF>',<-1>,3:1]
`,errors:"",grammarName:"L",grammar:`lexer grammar L;
T: ~'\\n'+;
SEPARATOR: '\\n';`,showDFA:!1,showDiagnosticErrors:!1,traceATN:!1,predictionMode:"LL",buildParseTree:!0,path:o.path}}static getLineSeparatorCrLfDescriptor(){return{testType:0,name:"LineSeparatorCrLf",notes:"",input:`1\r
2\r
3`,output:`[@0,0:0='1',<1>,1:0]
[@1,1:2='\\r\\n',<2>,1:1]
[@2,3:3='2',<1>,2:0]
[@3,4:5='\\r\\n',<2>,2:1]
[@4,6:6='3',<1>,3:0]
[@5,7:6='<EOF>',<-1>,3:1]
`,errors:"",grammarName:"L",grammar:`lexer grammar L;
T: ~'\\r'+;
SEPARATOR: '\\r\\n';`,showDFA:!1,showDiagnosticErrors:!1,traceATN:!1,predictionMode:"LL",buildParseTree:!0,path:o.path}}static getLargeLexerDescriptor(){let t="L",n="";n+="lexer grammar "+t+`;
`,n+=`WS: [ \\t\\r\\n]+ -> skip;
`;for(let r=0;r<4e3;r++)n+="KW"+r+" : 'KW' '"+r+`';
`;return{testType:0,name:"LargeLexer",notes:`This is a regression test for antlr/antlr4#76 "Serialized ATN strings
should be split when longer than 2^16 bytes (class file limitation)"
https://github.com/antlr/antlr4/issues/76`,input:"KW400",output:`[@0,0:4='KW400',<402>,1:0]
[@1,5:4='<EOF>',<-1>,1:5]
`,errors:"",grammarName:t,grammar:n.toString(),showDFA:!1,showDiagnosticErrors:!1,traceATN:!1,predictionMode:"LL",buildParseTree:!0,path:o.path}}static getAtnStatesSizeMoreThan65535Descriptor(){let t="_".repeat(70),n="L",r="";r+="lexer grammar "+n+`;
`,r+=`
`;let s="",i="",c,l=-2;for(let g=0;g<1024;g++){let k=`T_${this.padZero(g,6)}`,d=k+t;r+=k+": '"+d+`';
`,s+=d+`
`,c=l+2,l+=d.length+1,i+="[@"+g+","+c+":"+l+"='"+d+"',<"+(g+1)+">,"+(g+1)+`:0]
`}return r+=`
`,r+=`WS: [ \\t\\r\\n]+ -> skip;
`,c=l+2,l=c-1,i+="[@1024,"+c+":"+l+`='<EOF>',<-1>,1025:0]
`,{testType:0,name:"AtnStatesSizeMoreThan65535",notes:"Regression for https://github.com/antlr/antlr4/issues/1863",input:s.toString(),output:i.toString(),errors:"",grammarName:n,grammar:r.toString(),showDFA:!1,showDiagnosticErrors:!1,traceATN:!1,predictionMode:"LL",buildParseTree:!0,skipTargets:new Set(["CSharp","Python3","Go","PHP","Swift","JavaScript","TypeScript","Dart"]),path:o.path}}static getMultiTokenAlternativeDescriptor(){let t="r1: ",n="",r="",s="";for(let l=0;l<64;l++){let g="T"+l;t+=g,l<63?t+=" | ":t+=";",n+=g+": '"+g+`';
`,r+=g+" ",s+=g}let i="T64";n+=i+": '"+i+`';
`,r+=i+" ",s+=i+`
`;let c=`grammar P;
r: (r1 | T64)+ EOF {<writeln("$text")>};
`+t+`
`+n+`
WS: [ ]+ -> skip;`;return{testType:1,name:"MultiTokenAlternative",notes:"https://github.com/antlr/antlr4/issues/3698, https://github.com/antlr/antlr4/issues/3703",input:r.toString(),output:s,errors:"",startRule:"r",grammarName:"P",grammar:c,showDFA:!1,showDiagnosticErrors:!1,traceATN:!1,predictionMode:"LL",buildParseTree:!0,path:o.path}}static padZero(e,t){return e.toString().padStart(t,"0")}};var I=class o{static sections=new Set(["notes","type","grammar","slaveGrammar","start","input","output","errors","flags","skip"]);static parse(e,t,n){let r=null,s="",i=[],c=t.split(/\r?\n/);for(let h of c){let y=!1,a=null;h.startsWith("[")&&h.length>2&&(a=h.substring(1,h.length-1),y=o.sections.has(a)),y?(r!==null&&i.push([r,s]),r=a,s=""):s+=h+`
`}i.push([r??"",s]);let l="",g=0,k="",d="",u=[],m="",S="",T="",O="",x=!1,G=!1,v=!1,w="LL",R=!0,U;for(let h of i){let y=h[0],a="";switch(h[1]!==null&&(a=h[1].trim()),a.startsWith('"""')?a=a.replace(/"""/g,""):a.indexOf(`
`)>=0&&(a=a+`
`),y){case"notes":{l=a;break}case"type":{g=N[a];break}case"grammar":{d=o.getGrammarName(a.split(`
`)[0]),k=a;break}case"slaveGrammar":{let W=o.getGrammarName(a.split(`
`)[0]);u.push([W,a]);break}case"start":{m=a;break}case"input":{S=a;break}case"output":{T=a;break}case"errors":{O=a;break}case"flags":{let W=a.split(`
`);for(let ee of W){let z=ee.split("=",2);switch(z[0]){case"showDFA":{x=!0;break}case"showDiagnosticErrors":{G=!0;break}case"traceATN":{v=!0;break}case"predictionMode":{w=z[1];break}case"notBuildParseTree":{R=!1;break}default:}}break}case"skip":{U=new Set(a.split(`
`));break}default:throw new Error("Unknown descriptor section ignored: "+y)}}return{testType:g,name:e,notes:l,input:S,output:T,errors:O,startRule:m,grammarName:d,grammar:k,slaveGrammars:u,showDiagnosticErrors:G,traceATN:v,showDFA:x,predictionMode:w,buildParseTree:R,skipTargets:U,path:n}}static getGrammarName(e){let t=e.indexOf("grammar ");if(t<0)return"<unknown grammar name>";t+=8;let n=e.indexOf(";");return e.substring(t,n)}};var ae=process.env.GITHUB_ACTIONS==="true",E=class{constructor(e,t,n,r,s){this.basePath=e;this.configPath=t;this.config=n;this.silent=r;this.verbose=s}testDescriptors=new Map;stringRenderer=new oe;testCount=0;generate(){return this.readDescriptorsFromDisk(),this.addCustomDescriptors(),this.silent||console.log(`Found ${this.testCount} tests.
`),this.writeTestFiles()}readDescriptorsFromDisk(){let e=f(this.basePath,"resources/descriptors");K(e).forEach(t=>{if(re(f(e,t)).isDirectory()){let r=t;if(!r.startsWith(".")){let s=[],i=f(e,r);K(i).forEach(c=>{if(!c.startsWith(".")){let l=c.replace(".txt",""),g=J(f(i,c),{encoding:"utf-8"});s.push(I.parse(l,g,c)),++this.testCount}}),this.testDescriptors.set(r,s)}}}),this.silent||console.log("Test descriptors loaded.")}addCustomDescriptors(){for(let[e,t]of C.descriptors)this.testCount+=t.length,this.testDescriptors.has(e)?this.testDescriptors.get(e)?.push(...t):this.testDescriptors.set(e,t);this.silent||console.log("Custom test descriptors loaded.")}writeTestFiles(){let e=0;for(let[t,n]of this.testDescriptors){let r=f(this.configPath,this.config.targetPath??"tests",t);for(let s of n){if(!this.isTestIncluded(t,s.name))continue;if(++e,!this.silent){let v=`Processing (${Math.round(1e4*e/this.testCount)/100}%): ${t} > ${s.name}`;this.updateLine(ne.green(v))}let i=f(r,s.name);te(i,{recursive:!0});let c=new ie(f(this.configPath,this.config.grammarTemplateFile),"utf-8","<",">");c.registerRenderer(String,this.stringRenderer);let l=s.slaveGrammars;if(l)for(let v of l){let w=new B("<",">");w.registerRenderer(String,this.stringRenderer),w.importTemplates(c);let R=new $(w,v[1]);j(f(i,v[0]+".g4"),R.render(),{encoding:"utf-8"})}let g=new B("<",">");g.importTemplates(c),g.registerRenderer(String,this.stringRenderer);let d=new $(g,s.grammar).render(),u,m,S;s.testType===1||s.testType===3?(u=s.grammarName+"Lexer",m=s.grammarName+"Parser",S=!0):(u=s.grammarName,S=!1);let T;u&&m?m?T=m.endsWith("Parser")?m.substring(0,m.length-6):m:u&&(T=u.endsWith("Lexer")?u.substring(0,u.length-5):u):m!==void 0?T=m:T=u;let x={grammarStr:d,grammarName:T,parserName:m,lexerName:u,useListener:S,useVisitor:S};if(!this.generateParserFiles(i,x,s))return!1}}return!0}generateANTLRFilesInWorkDir(e,t,n,r){r.push("-Dlanguage="+t),r.includes("-o")||(r.push("-o"),r.push(e)),r.includes("--lib")||(r.push("--lib"),r.push(e)),r.includes("--encoding")||(r.push("--encoding"),r.push("UTF-8")),r.push(f(e,n));let s=new se(r);this.verbose||s.errorManager.addListener({errorManager:s.errorManager,info:()=>{},warning:()=>{},error:i=>{this.silent||console.error(i)}});try{s.processGrammarsOnCommandLine()}catch(i){return this.silent||console.error(i),!1}return s.errorManager.errors===0}consolidate(e){return e=e.replace(/\\/g,"\\\\"),e=e.replace(/\n/g,"\\n"),e=e.replace(/\r/g,"\\r"),e=e.replace(/"/g,'\\"'),e}writeTestFile(e,t,n){let r=J(f(this.configPath,this.config.specTemplateFile),{encoding:"utf-8"}),s=new $(r);s.add("grammarName",t.grammarName),s.add("lexerName",t.lexerName),s.add("parserName",t.parserName),s.add("parserStartRuleName",n.startRule??""),s.add("showDiagnosticErrors",n.showDiagnosticErrors),s.add("traceATN",n.traceATN),s.add("profile",!1),s.add("showDFA",n.showDFA),s.add("useListener",t.useListener),s.add("useVisitor",t.useVisitor),s.add("predictionMode",n.predictionMode),s.add("buildParseTree",n.buildParseTree),s.add("input",this.consolidate(n.input)),s.add("expectedOutput",this.consolidate(n.output)),s.add("expectedErrors",this.consolidate(n.errors)),s.add("testName",n.name);let i=n.skipTargets?.has(this.config.language)?1:0,c="";this.config.testAnnotations&&this.config.testAnnotations.length>0&&(c=this.config.testAnnotations[i]??""),s.add("testAnnotation",c);let l=this.config.testFileName??"Test";j(f(e,`${l}.${this.config.targetExtension}`),s.render())}generateParserFiles(e,t,n){let r=[];t.useVisitor&&r.push("-v"),L.mkdir(e);let s=n.grammarName+".g4";return L.writeFile(e,s,t.grammarStr),this.generateANTLRFilesInWorkDir(e,this.config.language,s,r)?(this.writeTestFile(e,t,n),j(f(e,"input"),n.input),!0):!1}matchesPattern(e,t){for(let n of t)if(e.match(n))return!0;return!1}isTestIncluded(e,t){let{groupIncludes:n=[],groupExcludes:r=[],testIncludes:s=[],testExcludes:i=[]}=this.config;return!(r.length>0&&this.matchesPattern(e,r)||n.length>0&&!this.matchesPattern(e,n)||i.length>0&&this.matchesPattern(t,i)||s.length>0&&!this.matchesPattern(t,s))}clearLine(){H.cursorTo(process.stdout,0),H.clearLine(process.stdout,1)}updateLine(e){if(ae){console.log(e);return}this.clearLine(),process.stdout.write(e)}};var pe=_(M(import.meta.url)),ue=A(pe,"../resources/config-schema.json"),Z=JSON.parse(Y(ue,{encoding:"utf-8"})),me=o=>{if(o&&X(o)){let e=Y(o,{encoding:"utf-8"}),t=JSON.parse(e),r=new le({allErrors:!0,verbose:!0}).compile(Z);if(!r(t)){console.log(`
Found config validation errors in ${o}
`);let i=ce.default(Z,t,r.errors??[],{json:e});console.log(i+`
`),process.exit(1)}return t}console.error(`
Config file ${o} not found.
`),process.exit(1)},fe=performance.now();Q.option("-c, --config <path>","Path to a JSON file containing the formatting options to use.").option("-s, --silent","Suppress all output including errors.").option("-v, --verbose","Print additional information.").version("antlr-tgen 1.0.0").parse();var b=Q.opts();b.silent||(console.log(q.bold(`
antlr-tgen, the antlr-ng test generator
`)),console.log("Processing options..."));var F=M(_(import.meta.url));for(;!X(A(F,"package.json"))&&F!=="/";)F=M(F);var V=A(process.cwd(),M(b.config)),P=me(b.config),de=new E(F,V,P,b.silent??!1,b.verbose??!1),he=de.generate();console.log();b.silent||console.log(q.blue("Copying files..."));if(P.files&&P.files.length>0)for(let o of P.files){let e=ge(A(V,o.sourcePattern));for(let t of e)b.verbose&&console.log(`	${t} -> ${o.targetPath}`),L.copyFileOrFolder(t,A(V,o.targetPath??P.targetPath??"./tests"))}b.silent||console.log(`
Done in ${Math.round((performance.now()-fe)/1e3)}s
`);he&&process.exit(0);process.exit(1);