tsmaz/dist/tsmaz.cjs.min.js

TypeScript port of the smaz string compression library

'use strict';Object.defineProperty(exports,"__esModule",{value:!0});function makeTrie(a){let c={chars:{},code:void 0};for(let b=0;b<a.length;b+=1){let d=a[b],e=c;for(let a=0;a<d.length;a+=1){let b=d[a];void 0===e.chars[b]&&(e.chars[b]={chars:{},code:void 0});e=e.chars[b]}e.code=b}return c}let EMPTY_STRING="",EMPTY_UINT8_ARRAY=new Uint8Array(0);
class Smaz{constructor(a){this.buffer=new Uint8Array(3E4);this.verbatim=new Uint8Array(255);this.codebook=a;this.trie=makeTrie(a)}compress(a){a=this.inplaceCompress(a);return 0===a?EMPTY_UINT8_ARRAY:this.buffer.slice(0,a)}UNSAFE_compress(a){a=this.inplaceCompress(a);return 0===a?EMPTY_UINT8_ARRAY:this.buffer.subarray(0,a)}decompress(a){if(0===a.byteLength)return EMPTY_STRING;let c=EMPTY_STRING,b=0;for(;b<a.byteLength;)254===a[b]?(c+=String.fromCharCode(a[b+1]),b+=2):255===a[b]?(c+=String.fromCharCode.apply(null,
a.subarray(b+2,b+a[b+1]+2)),b+=a[b+1]+2):(c+=this.codebook[a[b]],b+=1);return c}getCompressedSize(a){if(0===a.length)return 0;let c=0,b=0,d=0;for(;d<a.length;){let e=-1,f=-1,g=this.trie;for(let b=d;b<a.length;b+=1){g=g.chars[a[b]];if(void 0===g)break;void 0!==g.code&&(f=g.code,e=b+1)}-1===f?(b++,d++,255===b&&(c+=2+b,b=0)):(0!==b&&(c+=2+(1===b?0:b),b=0),c++,d=e)}0!==b&&(c+=2+(1===b?0:b));return c}inplaceCompress(a){if(0===a.length)return 0;let c=0,b=0,d=0;for(;d<a.length;){let e=-1,f=-1,g=this.trie;
for(let b=d;b<a.length;b+=1){g=g.chars[a[b]];if(void 0===g)break;void 0!==g.code&&(f=g.code,e=b+1)}-1===f?(this.verbatim[b++]=a.charCodeAt(d++),255===b&&(c=this.flushVerbatim(b,c),b=0)):(0!==b&&(c=this.flushVerbatim(b,c),b=0),this.buffer[c++]=f,d=e)}0!==b&&(c=this.flushVerbatim(b,c));return c}flushVerbatim(a,c){if(1===a)this.buffer[c++]=254,this.buffer[c++]=this.verbatim[0];else{this.buffer[c++]=255;this.buffer[c++]=a;for(let b=0;b<a;b+=1)this.buffer[c++]=this.verbatim[b]}return c}}
class StringCounter{constructor(){this.map=new Map}get(a){a=this.map.get(a);return void 0===a?0:a}incr(a){this.map.set(a,this.get(a)+1)}decr(a){let c=this.get(a);0===c?this.map.delete(a):this.map.set(a,c-1)}entries(){return this.map.entries()}}function extractNgrams(a){if(0===a.length)return[];let c=[],b=a.length;for(let d=0;d<b-1;d+=1){let e=b-d;for(let b=1;b<=e;b+=1)c.push(a.slice(d,d+b))}return c}function addCounts(a,c){for(let b of a)for(let a of extractNgrams(b))c.incr(a)}
function delCounts(a,c){for(let b of a)for(let a of extractNgrams(b))c.decr(a)}function getNextBestSubstring(a){let c=0,b="";for(let [d,e]of a.entries())1<e&&1<d.length&&(a=e*Math.pow(d.length,2.1),a>c&&(b=d,c=a));return b}function getCompressionRatio(a,c){a=new Smaz(a);let b=0,d=0;for(let e of c)b+=a.getCompressedSize(e),d+=e.length;return b/d*100}
function fineTuneCodebook(a,c,b){let d=getCompressionRatio(a,c),e=0;for(let f of b){if(254>a.length){a.push(f);console.log(`Codebook not full, adding : ${f} = ${getCompressionRatio(a,c)}`);continue}if(3<=e){console.log("Stopping optimization process after 3 rounds");break}b=100;let g=-1;console.log("?",f);for(let d=0;d<a.length;d+=1){let e=a[d];a[d]=f;let l=getCompressionRatio(a,c);a[d]=e;l<b&&(b=l,g=d)}b<d?(console.log(`replacing ${a[g]} with ${f} = ${b}% (-${(d-b).toFixed(3)}%)`),a[g]=f,d=b,e=0):
e+=1}}
function generate(a){let c=[];var b=new StringCounter;console.log("Counting substrings.");var d=new Set,e=[...a];addCounts(e,b);console.log("Creating codebook.");for(var f=0;254>f;f+=1){let g=getNextBestSubstring(b);if(0===g.length){console.log("No more strings",f);break}c.push(g);console.log(`+ ${g} = ${getCompressionRatio(c,a)}%`);let h=[],k=[];for(let a of e)if(!1===d.has(a)&&!0===a.includes(g)){k.push(a);for(let b of a.split(g))0!==b.length&&h.push(b)}addCounts(h,b);for(let a of h)e.push(a);delCounts(k,
b);for(let a of k)d.add(a)}for(let {n:e,candidates:f}of[{n:6,candidates:10},{n:5,candidates:10},{n:4,candidates:10},{n:3,candidates:10},{n:2,candidates:10}]){d=[];for(let [a,c]of b.entries())a.length===e&&d.push([a,c]);d.sort((a,b)=>b[1]-a[1]);console.log("Fine-tune codebook with ngrams",d);fineTuneCodebook(c,a,d.map(([a])=>a).slice(0,f))}b=new StringCounter;for(d=0;d<a.length;d+=1)for(e=a[d],f=0;f<e.length;f+=1)b.incr(e[f]);b=Array.from(b.entries()).sort((a,b)=>b[1]-a[1]);console.log("Fine-tune codebook with letters",
b);fineTuneCodebook(c,a,b.map(([a])=>a));return c}
let dictionary=' ;the;e;t;a;of;o;and;i;n;s;e ;r; th; t;in;he;th;h;he ;to;\r\n;l;s ;d; a;an;er;c; o;d ;on; of;re;of ;t ;, ;is;u;at;   ;n ;or;which;f;m;as;it;that;\n;was;en;  ; w;es; an; i;f ;g;p;nd; s;nd ;ed ;w;ed;http://;https://;for;te;ing;y ;The; c;ti;r ;his;st; in;ar;nt;,; to;y;ng; h;with;le;al;to ;b;ou;be;were; b;se;o ;ent;ha;ng ;their;";hi;from; f;in ;de;ion;me;v;.;ve;all;re ;ri;ro;is ;co;f t;are;ea;. ;her; m;er ; p;es ;by;they;di;ra;ic;not;s, ;d t;at ;ce;la;h ;ne;as ;tio;on ;n t;io;we; a ;om;, a;s o;ur;li;ll;ch;had;this;e t;g ; wh;ere; co;e o;a ;us; d;ss; be; e;s a;ma;one;t t;or ;but;el;so;l ;e s;s,;no;ter; wa;iv;ho;e a; r;hat;s t;ns;ch ;wh;tr;ut;/;have;ly ;ta; ha; on;tha;-; l;ati;en ;pe; re;there;ass;si; fo;wa;ec;our;who;its;z;fo;rs;ot;un;im;th ;nc;ate;ver;ad; we;ly;ee; n;id; cl;ac;il;rt; wi;e, ; it;whi; ma;ge;x;e c;men;.com'.split(";"),SMAZ=
null;function decompress(a){null===SMAZ&&(SMAZ=new Smaz(dictionary));return SMAZ.decompress(a)}function compress(a){null===SMAZ&&(SMAZ=new Smaz(dictionary));return SMAZ.compress(a)}function getCompressedSize(a){null===SMAZ&&(SMAZ=new Smaz(dictionary));return SMAZ.getCompressedSize(a)}exports.Smaz=Smaz;exports.compress=compress;exports.decompress=decompress;exports.generate=generate;exports.getCompressedSize=getCompressedSize;