treebank-tokenizer/lib/index.js

NLTK TreebankTokenizer Ported from Python to JS

function t(o,e){let i=0,r=[],s;for(let n of o){if(s=e.indexOf(n,i),s===-1)throw Error(`substring ${n} not found in "${e}"`);i=s+n.length,r.push([s,i])}return r}var a=class{constructor(){this.CONTRACTIONS2=[/\b(can)(not)\b/,/\b(d)('ye)\b/,/\b(gim)(me)\b/,/\b(gon)(na)\b/,/\b(got)(ta)\b/,/\b(lem)(me)\b/,/\b(more)('n)\b/,/\b(wan)(na)(?=s)/];this.CONTRACTIONS3=[/ ('t)(is)\b/,/ ('t)(was)\b/];this.CONTRACTIONS4=[/\b(whad)(dd)(ya)\b/,/\b(wha)(t)(cha)\b/]}},l=class{constructor(){this.STARTING_QUOTES=[[/^\"/,"``"],[/(``)/," $1 "],[/([ \(\[{<])(\"|\'{2})/,"$1 `` "]];this.PUNCTUATION=[[/([:,])([^\d])/g," $1 $2"],[/([:,])$/g," $1 "],[/\.\.\./g," ... "],[/([;@#$%&])/g," $1 "],[/([^\.])(\.)([\]\)}>\"\']*)\s*$/,"$1 $2$3 "],[/([?!])/g," $1 "],[/([^'])' /g,"$1 ' "]];this.PARENS_BRACKETS=[/([\]\[\(\){}<>])/g," $1 "];this.CONVERT_PARENTHESES=[[/\(/,"-LRB-"],[/\)/,"-RRB-"],[/\[/,"-LSB-"],[/\]/,"-RSB-"],[/{/,"-LCB-"],[/}/,"-RCB-"]];this.DOUBLE_DASHES=[/--/," -- "];this.ENDING_QUOTES=[[/''/," '' "],[/\"/," '' "],[/([^' ])('[sS]|'[mM]|'[dD]|') /,"$1 $2 "],[/([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) /,"$1 $2 "]];this._contractions=new a;this.CONTRACTIONS2=this._contractions.CONTRACTIONS2.map(e=>new RegExp(e,"i"));this.CONTRACTIONS3=this._contractions.CONTRACTIONS3.map(e=>new RegExp(e,"i"))}tokenize(e,i=!1){for(let n of this.STARTING_QUOTES){var[r,s]=[...n];e=e.replace(r,s)}for(let n of this.PUNCTUATION){var[r,s]=[...n];e=e.replace(r,s)}var[r,s]=[...this.PARENS_BRACKETS];if(e=e.replace(r,s),i)for(let n of this.CONVERT_PARENTHESES){var[r,s]=[...n];e=e.replace(r,s)}var[r,s]=[...this.DOUBLE_DASHES];e=e.replace(r,s),e=" "+e+" ";for(let n of this.ENDING_QUOTES){var[r,s]=[...n];e=e.replace(r,s)}for(let n of this.CONTRACTIONS2)e=e.replace(n," $1 $2 ");for(let n of this.CONTRACTIONS3)e=e.replace(n," $1 $2 ");return e.trim().split(/\s+/)}span_tokenize(e){let i=this.tokenize(e),r;if(e.includes('"')||e.includes("''")){let s=[...e.matchAll(/``|'{2}|\"/g)].map(n=>n[0]);r=i.map(n=>['"',"``","''"].indexOf(n)!==-1?s.splice(0,1)[0]:n)}else r=i;return t(r,e)}tokenize_sents(e){return e.map(i=>this.tokenize(i))}*span_tokenize_sents(e){for(let i of e)yield this.span_tokenize(i)}};module.exports=l;
//# sourceMappingURL=index.js.map