@zed.cwt/downloader/ZEDQuery.Downloader.js

ZEDQuery.Downloader

/*!
	Downloader 0.3.0

	In node

	Copyright@ZED.CWT, 2013-2018. All Rights Reserved !
	Mailto : ZEDCWT@zedcwt.com
*/

'use strict';var e,t=!e,a=!t,r=null,n=+e,i=require('@zed.cwt/zedquery'),s=require('fs'),o=require('path'),l=require('request').defaults({timeout:2e4}),p=require('url'),d=i.Observable,f=d.empty(),m=i.global.Buffer,u='Req',h='Obs',g='Stream',M='Path',c='Untitled',w='Fresh',S='Only200',b='ForceRange',N='AutoUnlink',P='Suffix',v='SuffixMeta',O='Interval',y='Delay',T=i.KeyGen(),j=i.KeyGen(),q=i.ReduceToObject(y,300,O,300,P,'.dl',v,'.meta'),F='Path',R='Size',x='Connected',D='Data',$='Drain',B='Done',C='Die',I=0,z=1,A=2,U=3,E=4,J=5,k=6,G=7,H=8,K=9,L=10,Q=11,V=12,W=13,X=14,Y=d.wrapNode(s.readdir),Z=d.wrapNode(s.open),_=d.wrapNode(s.write),ee=d.wrapNode(s.close),te=d.wrapNode(s.ftruncate),ae=d.wrapNode(s.rename),re=d.wrapNode(s.readFile),ne=d.wrapNode(s.unlink),ie=e=>d.create((r,n)=>(n=l(e).on('response',e=>{n.abort(),r.data([t,e]).finish()}).on('error',e=>r.data([a,e]).finish()),()=>n.abort())),se=d.wrapNode(i.FineName),oe={},le=e=>[oe,e],pe=e=>i.isArray(e)&&e[0]===oe,de=e=>((e=new Error(e)).name='DownloaderError',e),fe=e=>de('Bad meta | '+e),me=e=>de('Status not satisfied | '+e),ue=(e,t)=>de(`Not enough | ${e} < `+t),he=(e,t,a)=>de(`Size changed | ${e} - ${t} => `+a),ge=e=>e[P],Me=e=>e[P]+e[v],ce=(e,t)=>((t=(e=i.isObject(e)?i.Merge({},e):{url:e}).headers=i.Merge({},e.headers)).Host||t.host||(t=e.uri||e.url,i.isObject(t)||(t=p.parse(t)),(t=t.host)&&(e.headers.Host=t)),e),we=e=>Number(e&&e.headers['content-length']),Se=(e,t)=>(t=i.match(/filename\s*=\s*("[^"]+"|'[^']+'|[^;\s]+)/,e.headers['content-disposition']||'')[1])?i.Code.UTF8To16(unescape(t.replace(/^['"]|['"]$/g,''))):r,be=(e,a,r)=>a[K]||(r?a[K]=t:(r=e[u].url||e[u].url,i.isString(r)||(r=r.href||r.path)),a[H]=i.SafeFileName(r||'Untitled#'+i.Code.MD5(Math.random()))),Ne=(e,t)=>{var a=e[c]?o.join(e[M],t[H]):e[M],r=o.dirname(a),n=o.extname(a=o.basename(a)),i=0,s=(a=a.slice(0,-n.length))+(n+=ge(e));return Y(r).map(l=>{for(l=new Set(l);l.has(s)||t[J]&&l.has(s+e[v]);++i)s=a+` [${i}]`+n;return[s=o.join(r,s.slice(0,-ge(e).length)),s+ge(e),t[J]&&s+Me(e)]})},Pe=1500,ve=400,Oe=e=>{for(var t=e[X],a=i.now()-Pe,r=0;r<t.length&&t[r]<a;r+=2);return(r=i.max(0,i.min(i.max(r,t.length-ve),t.length-2)))&&t.splice(0,r),a=(t[t.length-1]-t[1])/(i.now()-t[0]),i.isFinite(a)?a:0},ye=(e,r,n)=>i.Merge(a,t,{headers:{Range:`bytes=${r}-${n}`}},e),Te=e=>t=>(t=m.from(i.OTJ([e[Q],e[G],e[I]]),'utf8'),_(e[E],t,0,t.length,0)),je=e=>(t,a)=>(a=_(e[U],t,0,t.length,e[L]).tap(()=>e[X].push(i.now(),e[Q]=t)),t=e[L]+=t.length,a),qe=(e,a,r)=>i.Merge(t,e[T],{Pending:a[L],PendingP:a[L]/a[G]||0,Saved:a[Q],SavedP:a[Q]/a[G]||0,Past:i.now()-a[V],Speed:r=Oe(a),Rest:(a[G]-a[Q])/r}),Fe=(e,a,n)=>{var s,o=a[k],l=a[H],p=a[K];return n.flatMap(()=>(a[X]=[i.now(),a[Q]],i.Merge(t,e[T],{Total:a[G],Start:a[V]=i.now(),Begin:a[Q],BeginP:a[Q]/a[G]||0,Pending:a[L],PendingP:a[L]/a[G]||0,Saved:a[Q],SavedP:a[Q]/a[G]||0}),e[O]&&(a[W]=setInterval(()=>{qe(e,a),e[j].emit(D,e[T])},e[O])),s=[],e[c]&&(!l&&a[H]||!p&&a[K])&&s.push(Ne(e,a).flatMap(([t,n,i])=>d.merge(ae(a[z],a[z]=n),i?ae(a[A],a[A]=i):f).tap(r,r,()=>e[j].emit(F,t)))),o||a[k]&&s.push(te(a[U],a[G])),e[j].emit(x),d.from(s).online(1)))},Re=(e,t)=>(e[g]?d.just([a]):ie(ye(e[u],0,1))).flatMap(([a,r])=>(t[J]=a&&'2'===r.headers['content-length'],be(e,t,a&&Se(r)),e[b]&&(!a||206!==r.statusCode)&&i.Throw(r),t[I]=e[c]?o.join(e[M],t[H]):e[M],Ne(e,t))).flatMap(([a,r,n])=>d.merge(Z(t[z]=r,'w+').tap(e=>t[U]=e),n?Z(t[A]=n,'w+').tap(e=>t[E]=e):f).finish().flatMap(()=>(e[j].emit(F,a),t[Q]=t[L]=0,(n?xe:De)(e,t)))).finish().flatMap(Ie(e,t)),xe=(e,t,a)=>d.merge(Fe(e,t,a=i.Repeater()),$e(e,t,a).flatMapOnline(1,je(t)).throttle(e[y]).flatMap(Te(t))),De=(e,t,a)=>d.merge(Fe(e,t,a=i.Repeater()),$e(e,t,a).flatMapOnline(1,je(t))),$e=(e,t,a)=>t[Q]<t[G]||!t[k]?d.create((r,n)=>(n=l(t[J]?ye(e[u],t[Q],''):e[u]).on('response',n=>{e[S]&&!/^2/.test(n.statusCode)&&r.error(le(me(n.statusCode))),t[k]?t[G]===t[Q]+we(n)||r.error(he(t[G],t[Q],we(n))):(t[G]=we(n),(t[k]=t[G]==t[G])&&e[j].emit(R,t[G])),be(e,t,Se(n)),a.data().finish()}).on('data',e=>r.data(e)).on('complete',()=>t[L]<t[G]?r.error(ue(t[L],t[G])):r.finish(e[j].emit($))).on('error',e=>r.error(e)),()=>n.abort())):f,Be=(e,a)=>re(e[M]+Me(e),'utf8').flatMap(r=>(r=i.JTO(r.replace(/][^]*/,']')),i.isArray(r)&&3==r.length&&i.isNumber(r[0])&&i.isNumber(r[1])&&i.isString(r[2])||i.Throw(fe(i.OTJ(r))),a[I]=r[2],a[J]=t,a[k]=t,a[G]=r[1],a[L]=a[Q]=r[0],Z(a[z]=e[M]+ge(e),'r+'))).flatMap(t=>(a[U]=t,Z(a[A]=e[M]+Me(e),'r+'))).flatMap(t=>(a[E]=t,xe(e,a))).finish().flatMap(Ie(e,a)),Ce=(e,t)=>se(e[M]+ge(e),o.extname(e[M])+ge(e)).flatMap(e=>Z(t[z]=e,'w+')).flatMap(a=>(t[U]=a,e[j].emit(F,t[z]),t[G]=n,Fe(e,t,d.just()))).finish().flatMap(()=>e[h]).flatMapOnline(1,je(t)).finish().flatMap(Ie(e,t)),Ie=(e,t)=>()=>d.merge(d.just().tap(()=>{clearInterval(t[W]),i.isFinite(t[G])||(t[G]=t[Q]),qe(e,t),e[j].emit(D,e[T])}),i.isNumber(t[U])?ee(t[U]):f,i.isNumber(t[E])?ee(t[E]):f).finish().flatMap(()=>se(t[I])).flatMap(a=>ae(t[z],a).tap(()=>e[j].emit(F,a))).flatMap(()=>t[A]?ne(t[A]):f);module.exports=(e=>{var t,a=[],s=i.Emitter(),o=()=>s.emit(B),l=()=>Re(e,a).start(r,e=>p().start(()=>s.emit(C,pe(e)?e[1]:e)),o),p=()=>(a[W]&&clearInterval(a[W]),t&&t.end(),d.merge(i.isNumber(a[U])?ee(a[U]):f,i.isNumber(a[E])?ee(a[E]):f).flatMap(()=>e[N]&&!a[J]&&a[z]?ne(a[z]):f).all().finish());return(e=i.Merge({},e,q))[u]&&(e[u]=ce(e[u])),e[j]=s,t=e[h]?Ce(e,a).start(r,e=>p().start(()=>s.emit(C,e))):e[w]||e[c]||e[g]?l():Be(e,a).start(r,e=>p().start(()=>a[X]||pe(e)?s.emit(C,pe(e)?e[1]:e):t=l(a=[])),o),i.Merge(s,{Info:e[T]={Total:n,Start:0,Begin:0,BeginP:0,Pending:0,PendingP:0,Saved:0,SavedP:0,Past:0,Speed:0,Rest:1/0},Speed:()=>a[X]?Oe(a):0,Stop:()=>p().start(i.noop),StopAsync:p})});