tokenizr/dst/tokenizr.umd.js

String Tokenization Library for JavaScript

(function(a,u){typeof exports=="object"&&typeof module<"u"?u(exports):typeof define=="function"&&define.amd?define(["exports"],u):(a=typeof globalThis<"u"?globalThis:a||self,u(a.Tokenizr={}))})(this,(function(a){"use strict";const u=l=>l.charCodeAt(0).toString(16).toUpperCase(),m=(l,t)=>{const i=l.length,e=Math.max(t-20,0),n=Math.min(t+20,i),r=(s,h,_)=>s.substring(h,h+_).replaceAll(/\\/g,"\\\\").replaceAll(/\x08/g,"\\b").replaceAll(/\t/g,"\\t").replaceAll(/\n/g,"\\n").replaceAll(/\f/g,"\\f").replaceAll(/\r/g,"\\r").replaceAll(/[\x00-\x07\x0B\x0E\x0F]/g,o=>"\\x0"+u(o)).replaceAll(/[\x10-\x1F\x80-\xFF]/g,o=>"\\x"+u(o)).replaceAll(/[\u0100-\u0FFF]/g,o=>"\\u0"+u(o)).replaceAll(/[\u1000-\uFFFF]/g,o=>"\\u"+u(o));return{prologTrunc:e>0,prologText:r(l,e,t-e),tokenText:r(l,t,1),epilogText:r(l,t+1,n-(t+1)),epilogTrunc:n<i}};class c{constructor(t,i,e,n=0,r=0,s=0){this.type=t,this.value=i,this.text=e,this.pos=n,this.line=r,this.column=s}toString(t=(i,e)=>e){return`${t("type",this.type)} (value: ${t("value",JSON.stringify(this.value))}, text: ${t("text",JSON.stringify(this.text))}, pos: ${t("pos",this.pos.toString())}, line: ${t("line",this.line.toString())}, column: ${t("column",this.column.toString())})`}isA(t,i){return!(t!==this.type||i!==void 0&&i!==this.value)}}class p extends Error{constructor(t,i,e,n,r){super(t),this.name="ParsingError",this.message=t,this.pos=i,this.line=e,this.column=n,this.input=r}toString(){const t=m(this.input,this.pos),i=`line ${this.line} (column ${this.column}): `,e=t.prologTrunc?"...":'"',n=t.epilogTrunc?"...":'"',r=" ".repeat(i.length+e.length+t.prologText.length);return"Parsing Error: "+this.message+`
`+i+e+t.prologText+t.tokenText+t.epilogText+n+`
`+r+"^"}}class f{constructor(t){this._tokenizr=t,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}data(t,i){const e=this._data[t];return arguments.length===2&&(this._data[t]=i),e}info(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match?.[0]?.length??0}}push(t){return this._tokenizr.push(t),this}pop(){return this._tokenizr.pop()}state(t){return t!==void 0?(this._tokenizr.state(t),this):this._tokenizr.state()}tag(t){return this._tokenizr.tag(t),this}tagged(t){return this._tokenizr.tagged(t)}untag(t){return this._tokenizr.untag(t),this}repeat(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}reject(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}ignore(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}accept(t,i){return i=i??this._match?.[0],this._tokenizr._log(`    ACCEPT: type: ${t}, value: ${JSON.stringify(i)} (${typeof i}), text: "${this._match?.[0]??""}"`),this._tokenizr._pending.push(new c(t,i,this._match?.[0]??"",this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}stop(){return this._tokenizr._stopped=!0,this}}class w{constructor(){this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new f(this)}reset(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new f(this),this}error(t){return new p(t,this._pos,this._line,this._column,this._input)}debug(t){return this._debug=t,this}_log(t){this._debug&&console.log(`tokenizr: ${t}`)}input(t){if(typeof t!="string")throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}push(t){if(arguments.length!==1)throw new Error("invalid number of arguments");if(typeof t!="string")throw new Error('parameter "state" not a String');return this._log(`    STATE (PUSH): old: <${this._state[this._state.length-1]}>, new: <${t}>`),this._state.push(t),this}pop(){if(arguments.length!==0)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log(`    STATE (POP): old: <${this._state[this._state.length-1]}>, new: <${this._state[this._state.length-2]}>`),this._state.pop()}state(t){if(arguments.length===1){if(typeof t!="string")throw new Error('parameter "state" not a String');return this._log(`    STATE (SET): old: <${this._state[this._state.length-1]}>, new: <${t}>`),this._state[this._state.length-1]=t,this}else if(arguments.length===0)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}tag(t){if(arguments.length!==1)throw new Error("invalid number of arguments");if(typeof t!="string")throw new Error('parameter "tag" not a String');return this._log(`    TAG (ADD): ${t}`),this._tag[t]=!0,this}tagged(t){if(arguments.length!==1)throw new Error("invalid number of arguments");if(typeof t!="string")throw new Error('parameter "tag" not a String');return this._tag[t]===!0}untag(t){if(arguments.length!==1)throw new Error("invalid number of arguments");if(typeof t!="string")throw new Error('parameter "tag" not a String');return this._log(`    TAG (DEL): ${t}`),delete this._tag[t],this}before(t){return this._before=t,this}after(t){return this._after=t,this}finish(t){return this._finish=t,this}rule(t,i,e,n="unknown"){if(arguments.length===2&&typeof i=="function"?([i,e]=[t,i],t="*"):arguments.length===3&&typeof i=="function"&&([i,e,n]=[t,i,e],t="*"),typeof t!="string")throw new Error('parameter "state" not a String');if(!(i instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if(typeof e!="function")throw new Error('parameter "action" not a Function');if(typeof n!="string")throw new Error('parameter "name" not a String');const r=t.split(/\s*,\s*/g).map(_=>{const o=_.split(/\s+/g),d=o.filter(g=>!g.startsWith("#")),x=o.filter(g=>g.startsWith("#")).map(g=>g.slice(1));if(d.length!==1)throw new Error("exactly one state required");return{state:d[0],tags:x}});let s="y";i.multiline&&(s+="m"),i.dotAll&&(s+="s"),i.ignoreCase&&(s+="i"),i.unicode&&(s+="u");const h=new RegExp(i.source,s);return this._log(`RULE: configure rule (state: ${t}, pattern: ${h.source})`),this._rules.push({state:r,pattern:h,action:e,name:n}),this}_progress(t,i){const e=this._line,n=this._column,r=this._input;for(let s=t;s<i;s++){const h=r.charAt(s);h==="\r"?(this._line++,this._column=1,s+1<i&&r.charAt(s+1)===`
`&&s++):h===`
`?(this._line++,this._column=1):h==="	"?this._column+=8-(this._column-1)%8:this._column++}this._log(`    PROGRESS: characters: ${i-t}, from: <line ${e}, column ${n}>, to: <line ${this._line}, column ${this._column}>`)}_tokenize(){const t=()=>{this._eof||(this._finish!==null&&this._finish.call(this._ctx,this._ctx),this._eof=!0,this._pending.push(new c("EOF","","",this._pos,this._line,this._column)))};if(this._stopped||this._pos>=this._len){t();return}let i=!0;for(;i;){if(i=!1,this._debug){const e=m(this._input,this._pos),n=Object.keys(this._tag).map(r=>`#${r}`).join(" ");this._log(`INPUT: state: <${this._state[this._state.length-1]}>, tags: <${n}>, text: `+(e.prologTrunc?"...":'"')+`${e.prologText}<${e.tokenText}>${e.epilogText}`+(e.epilogTrunc?"...":'"')+`, at: <line ${this._line}, column ${this._column}>`)}for(let e=0;e<this._rules.length;e++){if(this._debug){const h=this._rules[e].state.map(_=>{let o=_.state;return _.tags.length>0&&(o+=" "+_.tags.map(d=>`#${d}`).join(" ")),o}).join(", ");this._log(`  RULE: state(s): <${h}>, pattern: ${this._rules[e].pattern.source}`)}let n=!1,r=this._rules[e].state.findIndex(h=>h.state===this._state[this._state.length-1]);if(r<0&&(r=this._rules[e].state.findIndex(h=>h.state==="*")),r>=0&&(n=this._rules[e].state[r].tags.every(_=>this._tag[_])),!n)continue;this._rules[e].pattern.lastIndex=this._pos;const s=this._rules[e].pattern.exec(this._input);if(s!==null&&s.index===this._pos)if(this._debug&&this._log("    MATCHED: "+JSON.stringify(s)),this._ctx._match=s,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._before!==null&&this._before.call(this._ctx,this._ctx,s,this._rules[e]),this._rules[e].action.call(this._ctx,this._ctx,s),this._after!==null&&this._after.call(this._ctx,this._ctx,s,this._rules[e]),this._stopped){t();return}else{if(this._ctx._reject)continue;if(this._ctx._repeat){i=!0;break}else if(this._ctx._ignore){if(this._progress(this._pos,this._rules[e].pattern.lastIndex),this._pos=this._rules[e].pattern.lastIndex,this._pos>=this._len){t();return}i=!0;break}else if(this._pending.length>0){this._progress(this._pos,this._rules[e].pattern.lastIndex),this._pos=this._rules[e].pattern.lastIndex,this._pos>=this._len&&t();return}else throw new Error('action of pattern "'+this._rules[e].pattern.source+'" neither rejected nor accepted any token(s)')}}}throw this.error("token not recognized")}token(){if(this._pending.length===0&&this._tokenize(),this._pending.length>0){const t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log(`TOKEN: ${t.toString()}`),t}return null}tokens(){const t=[];let i;for(;(i=this.token())!==null;)t.push(i);return t}peek(t){if(arguments.length===0&&(t=0),typeof t!="number"||t<0)throw new Error('parameter "offset" not a positive Number');for(;t>=this._pending.length;){const i=this._pending.length;if(this._tokenize(),this._eof||this._pending.length===i)break}if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log(`PEEK: ${this._pending[t].toString()}`),this._pending[t]}skip(t){t===void 0&&(t=1);for(let i=0;i<t;i++)if(this.token()===null)throw new Error("not enough tokens available for skip operation");return this}consume(t,i){if(this._pending.length===0&&this._tokenize(),this._pending.length===0)throw new Error("not enough tokens available for consume operation");const e=this.token();this._log(`CONSUME: ${e.toString()}`);const n=()=>{throw new p(`expected: <type: ${t}, value: ${JSON.stringify(i)} (${typeof i})>, found: <type: ${e.type}, value: ${JSON.stringify(e.value)} (${typeof e.value})>`,e.pos,e.line,e.column,this._input)};return(arguments.length===2&&!e.isA(t,i)||arguments.length===1&&!e.isA(t))&&n(),e}begin(){return this._log(`BEGIN: level ${this._transaction.length}`),this._transaction.unshift([]),this}depth(){if(this._transaction.length===0)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}commit(){if(this._transaction.length===0)throw new Error("cannot commit transaction -- no active transaction");const t=this._transaction.shift();return this._transaction.length>0&&(this._transaction[0]=this._transaction[0].concat(t)),this._log(`COMMIT: level ${this._transaction.length}`),this}rollback(){if(this._transaction.length===0)throw new Error("cannot rollback transaction -- no active transaction");const t=this._transaction.shift();return this._pending=t.concat(this._pending),this._log(`ROLLBACK: level ${this._transaction.length}`),this}alternatives(...t){let i=null,e=!1;const n=[];for(let r=0;r<t.length;r++){let s=0;try{this.begin(),s=this._transaction.length,i=t[r].call(this),this.commit(),e=!0;break}catch(h){for(;this._transaction.length>s;)this.rollback();h instanceof Error?(this._log(`EXCEPTION: ${h.message}`),n.push({ex:h,depth:this.depth()})):(this._log("EXCEPTION: alternative failed"),n.push({ex:new Error("alternative failed"),depth:this.depth()})),this.rollback();continue}}if(!e&&n.length>0)throw n.sort((r,s)=>s.depth-r.depth),n[0].ex;return i}static{this.Token=c}static{this.ParsingError=p}static{this.ActionContext=f}}a.ActionContext=f,a.ParsingError=p,a.Token=c,a.Tokenizr=w,Object.defineProperty(a,Symbol.toStringTag,{value:"Module"})}));