nyaa.si-client
Version:
<div align="center">
2 lines (1 loc) • 18.1 kB
JavaScript
import q from"axios";import{nanoid as $}from"nanoid";import z from"puppeteer";var E=(t=>(t.Puppeteer="puppeteer",t.Cheerio="cheerio",t))(E||{});var F=(i=>(i.Reserved="reserved",i.Listening="listening",i.Dead="dead",i))(F||{});import{EventEmitter as K}from"events";var L=class{emitter=new K;on(e,t){return this.emitter.on(e,t),this}emit(e,...t){return this.emitter.emit(e,...t)}off(e,t){return this.emitter.off(e,t),this}};import"puppeteer";var P=class{id;status="dead";timeout;runTime;api;client;constructor({runTime:e,timeout:t,client:i,api:a,id:n}){this.runTime=e,this.timeout=t,this.client=i,this.id=n,this.api=a,this.status="listening",this.client.emit("started",this),this.client.emit("listening",this)}async finishedTask(){this.runTime==="puppeteer"&&this.api.goto("about:blank"),await new Promise(e=>setTimeout(()=>e(),this.timeout)),this.status="listening",this.client.emit("listening",this)}stop(){I.session.clear(this.id),this.runTime==="puppeteer"&&(this.api.isClosed()||this.api.close()),this.client.emit("died",this),this.client.jobs.delete(this.id)}};import j from"user-agents";var k=class{constructor(e){this.options=e}sessions=new Map;create(e,t){let i=this.options?.defaultUserAgent??new j(this.options?.userAgentOptions??t).toString();return this.sessions.set(e,i),i}get(e){return this.sessions.get(e)}createOrGet(e,t){let i=this.get(e);return i||this.create(e,t)}clear(e){return this.sessions.delete(e)}};var I=class h{static session=new k;runTime="cheerio";browser;jobs=new Map;timeout=3e3;showNavigator=!1;concurrentJobs=1;requestQueue=[];host="https://nyaa.si";emitter=new L;on(e,t){this.emitter.on(e,t)}emit(e,...t){this.emitter.emit(e,...t)}constructor(e){e?.session&&(h.session=e.session),e?.showNavigator&&(this.showNavigator=e.showNavigator),e?.concurrentJobs&&(this.concurrentJobs=e.concurrentJobs),e?.runTime&&(this.runTime=e.runTime),this.setupQueueListener()}async initialize(){switch(this.runTime){case"cheerio":{this.setupCheerio();break}case"puppeteer":await this.setupPuppeteer()}}async setupPuppeteer(){return this.browser=await z.launch({waitForInitialPage:!0,acceptInsecureCerts:!0,headless:!this.showNavigator}),await Promise.all(Array.from({length:this.concurrentJobs},async()=>{let{page:e,pageId:t}=await this.newPage();new P({id:t,api:e,runTime:this.runTime,timeout:this.timeout,client:this})})),this}setupCheerio(){for(let e=0;e<this.concurrentJobs;e++){let{instance:t,instanceId:i}=this.newAxiosInstance(),a=new P({id:i,api:t,runTime:this.runTime,timeout:this.timeout,client:this});this.jobs.set(a.id,a)}return this}set setURL(e){this.host=e}getURL(){return this.host}newAxiosInstance(){let e=$(),t=h.session.create(e);return{instance:q.create({baseURL:this.host,headers:{"User-Agent":t},method:"GET"}),instanceId:e}}async newPage(){let t=await(await this.browser.createBrowserContext()).newPage(),i=t.mainFrame()._id,a=h.session.create(i);return await t.setUserAgent(a),await t.setViewport({width:1920,height:1080}),{page:t,pageId:i}}requestJob(){return new Promise(e=>{let t=Array.from(this.jobs.values()).find(i=>i.status==="listening");if(t)return t.status="reserved",this.emit("reserved",t),e(t);this.requestQueue.push(e)})}setupQueueListener(){this.on("listening",e=>{let t=this.requestQueue.shift();t&&(e.status="reserved",this.emit("reserved",e),t(e))})}};var O=class{scraper;search;data;constructor(e){this.scraper=e.scraper,this.data=e.data,this.search=e.search}getData(){return this.data}setData(e){this.data=e}async addNextPage(e=1){if(!this.data.metadata.hasNextPage)return this;let t=await this.scraper.search(this.search.content,{page:this.data.metadata.current+1,loadOnlyPage:!(e>1),filter:this.search.filter},this);return this.setData(t.data),this}async getNextPage(e=1){if(!this.data.metadata.hasNextPage)throw new Error("There is no next page for fetching information");return await this.scraper.search(this.search.content,{page:this.data.metadata.current+1,loadOnlyPage:!(e>1),filter:this.search.filter})}};import*as S from"cheerio";var w=(t=>(t.List="list",t.Details="details",t))(w||{}),C=(t=>(t.File="file",t.Folder="folder",t))(C||{});var A=class{job;url;loadAdditionalInfo;constructor({job:e,url:t,loadAdditionalInfo:i}){this.job=e,this.url=t,this.loadAdditionalInfo=i??!1}};var R=class h extends A{async extract(e){let t=await this.job.api(`${this.url}?${e}`),i=S.load(t.data),a=i(".torrent-list tbody tr"),n=[];for(let r of a){let d=i(r).find("td:nth-child(1) a").first().attr()?.title,c=i(r).find("td:nth-child(2) a:not(.comments)").first().attr()?.title,p=i(r).find("td:nth-child(2) a:not(.comments)").first().attr()?.href,f=i(r).find('td:nth-child(3) a[href$=".torrent"]').first().attr()?.href,s=i(r).find('td:nth-child(3) a[href^="magnet:"]').first().attr()?.href,o=i(r).find("td:nth-child(4)").first().text().trim(),m=parseInt(i(r).find("td:nth-child(5)").first().attr()?.["data-timestamp"]??""),b=parseInt(i(r).find("td:nth-child(6)").first().text().trim()),g=parseInt(i(r).find("td:nth-child(7)").first().text().trim()),x=parseInt(i(r).find("td:nth-child(8)").first().text().trim());d&&c&&s&&p&&f&&n.push({id:parseInt(p.match(/\/view\/(\d+)/)?.[1]??"0"),hash:s?.match(/magnet:\?xt=urn:btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})/)?.[1]??"",category:d,name:c,links:{magnet:s,page:p,torrent:f},size:o,timestamp:m,stats:{seeders:b,leechers:g,downloaded:x}})}let u=i("ul.pagination li"),l={hasPreviousPage:!0,hasNextPage:!0,current:0,total:0,timestamp:Date.now(),timeTaken:0};for(let r of u)switch(!0){case r.attribs.class?.includes("active"):{let c=i(r).find("a").first().text().trim()??this.url.match(/[?&]p=(\d+)/)?.[1]??"0";l.current=parseInt(c);break}case r.attribs.class?.includes("previous"):{r.attribs.class?.includes("disabled")&&(l.hasPreviousPage=!1);break}case r.attribs.class?.includes("next"):{r.attribs.class?.includes("disabled")&&(l.hasNextPage=!1);let d=new URL(i(r).find("a").attr()?.href??"",this.url).toString();l.nextPageLink=d,l.nextPage=h.extractPageNumber(new URL(d,this.url).toString())??0;break}case r.attribs.class?.includes("prev"):{let d=new URL(i(r).find("a").attr()?.href??"",this.url).toString();l.previousPageLink=d,l.previousPage=h.extractPageNumber(new URL(d,this.url).toString());break}default:{let d=l.total,c=parseFloat(i(r).find("a").text()??"0");d===void 0&&(l.total=c),Number(d)<c&&(l.total=c)}}return await this.job.finishedTask(),{type:"list",count:n.length,metadata:l,torrents:n}}async details(e,t){let i=await this.job.api(e),a=S.load(i.data),n={type:"details",description:void 0,submitter:void 0,information:void 0,files:void 0,comments:void 0};if(t===void 0||t===!0||t!==!1&&t?.submitter){let u=a('[title="User"]'),l=u.text(),r=u.attr()?.href;l&&r&&(n.submitter={name:l,url:new URL(r,this.url).toString()})}if((t===void 0||t===!0||t!==!1&&t?.information)&&(n.information=a("body > div > div:nth-child(1) > div.panel-body > div:nth-child(3) > div:nth-child(2)").text().trim()),t===!0||t!==!1&&t?.description){let u=a("torrent-description");switch(t){case!0:{let l=u.html();if(l){n.description=l;break}break}default:switch(t.description){case"text":{n.description=u.text().trim();break}default:{n.description=u.html();break}}}}if(t===void 0||t===!0||t!==!1&&t?.files){let u=a(".torrent-file-list");if(u){let l=h.parseTorrentComponents(u.html());n.files=l}}if(t===void 0||t===!0||t!==!1&&t?.comments){let u=a(".comment-panel"),l=[];for(let r of u){let d=a(r).find("img").first().attr("src")??"",c=a(r).find('[title="User"]').first().text().trim(),p=a(r).find('[title="User"]').first().parent().text().trim().includes("uploader")??!1,f=a(r).find("[markdown-text]").first().text().trim(),s=a(r).find("[data-timestamp-swap]").first(),o=s.data("timestamp"),m=s.text().replaceAll("UTC","").trim();!d||!c||!f||!o||!m||l.push({avatarURL:d,userName:c,message:f,publishDate:m,timestamp:parseInt(o),isUploader:p})}l.length>0&&(n.comments=l)}return await this.job.finishedTask(),n}static parseTorrentComponents(e){if(!e)return[];let t=S.load(e),i=[],a=t(e).children("ul li");for(let n of a){let u=t(n).children("a.folder").first();if(u.length>0){let l=u.text().trim();if(!l)return i;let r=t(n).html();if(!r)return i;let d=this.parseTorrentComponents(r);i.push({type:"folder",name:l,files:d})}else{let l=t(n).children("span.file-size").text().trim();if(!l)return i;let r=t(n).text().replaceAll(l,"");if(!r)return i;let d=l.replace("(","").replace(")","");i.push({type:"file",fileName:r,readableSize:d,sizeInBytes:this.parseSizeToBytes(d)})}}return i}static parseSizeToBytes(e){let t={B:1,KiB:1024,MiB:1048576,GiB:1073741824,TiB:1099511627776,PiB:0x4000000000000},i=/([\d.]+)\s*(B|KiB|MiB|GiB|TiB|PiB)/i,a=e.match(i);if(!a)throw new Error(`Invalid size format: ${e}`);let[,n,u]=a,l=parseFloat(n);if(!Object.prototype.hasOwnProperty.call(t,u))throw new Error(`Unknown unit: ${u}`);let r=t[u];return l*r}static extractPageNumber(e){let i=new URL(e).searchParams.get("p");return i?parseInt(i,10):void 0}};import{NodeHtmlMarkdown as G}from"node-html-markdown";var B={anime:["anime_music_video","english_translated","non_english_translated","raw"],audio:["lossless","lossy"],literature:["english_translated","non_english_translated","raw"],live_action:["english_translated","idol_promotional_video","non_english_translated","raw"],pictures:["graphics","photos"],software:["applications","games"]},N=(i=>(i.TextContent="textContent",i.Href="href",i.GetAttribute="getAttribute",i))(N||{}),H=(i=>(i[i.NoFilter=0]="NoFilter",i[i.NoRemakes=1]="NoRemakes",i[i.TrustedOnly=2]="TrustedOnly",i))(H||{});var _=new G,v=class extends A{async extract(e){return await this.waitForConnect(`${this.url}?${e}`),await this.job.api.evaluate((t,i)=>{function a({selector:s,subSelector:o,row:m}){let b=o?`${s} ${o}`:s;return m.querySelector(b)??void 0}function n(s){if(!s.element)return;let{element:o,type:m}=s;switch(m){case t.GetAttribute:return o.getAttribute(s.attributeName)??void 0;case t.TextContent:return o.textContent?.trim();case t.Href:return o.href}}let u=document.querySelector(".torrent-list");if(!u)throw new Error("Torrent list not available!");let r=Array.from(u.querySelectorAll("tbody tr")).map(s=>{let o=n({type:t.GetAttribute,element:a({row:s,selector:"td:nth-child(1)",subSelector:"a"}),attributeName:"title"}),m=n({type:t.GetAttribute,element:a({row:s,selector:"td:nth-child(2)",subSelector:"a:not(.comments)"}),attributeName:"title"}),b=n({type:t.Href,element:a({row:s,selector:"td:nth-child(2)",subSelector:"a:not(.comments)"})}),g=n({type:t.Href,element:a({row:s,selector:"td:nth-child(3)",subSelector:'a[href$=".torrent"]'})}),x=n({type:t.Href,element:a({row:s,selector:"td:nth-child(3)",subSelector:'a[href^="magnet:"]'})}),y=n({type:t.TextContent,element:a({row:s,selector:"td:nth-child(4)"})}),T=parseInt(n({type:t.GetAttribute,element:a({row:s,selector:"td:nth-child(5)"}),attributeName:"data-timestamp"})??"undefined"),D=parseInt(n({type:t.TextContent,element:a({row:s,selector:"td:nth-child(6)"})})??""),U=parseInt(n({type:t.TextContent,element:a({row:s,selector:"td:nth-child(7)"})})??""),M=parseInt(n({type:t.TextContent,element:a({row:s,selector:"td:nth-child(8)"})})??"");return{id:parseInt(b?.match(/\/view\/(\d+)/)?.[1]??""),hash:x?.match(/magnet:\?xt=urn:btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})/)?.[1],name:m,timestamp:T,size:y,category:o,links:b&&x&&g?{page:b,magnet:x,torrent:g}:void 0,stats:{seeders:D,leechers:U,downloaded:M}}}),d=document.querySelectorAll("ul.pagination"),c={hasPreviousPage:!0,hasNextPage:!0,current:0,total:0,timestamp:Date.now(),timeTaken:0};function p(s){let m=new URL(s).searchParams.get("p");return m?parseInt(m,10):null}for(let s of Array.from(d[0].querySelectorAll("li")))switch(!0){case s.classList.contains("active"):{let m=s.querySelector("a").childNodes[0].textContent?.trim()??window.location.href.match(/[?&]p=(\d+)/)?.[1]??"0";c.current=parseInt(m);break}case s.classList.contains("previous"):{s.classList.contains("disabled")&&(c.hasPreviousPage=!1);break}case s.classList.contains("next"):{s.classList.contains("disabled")&&(c.hasNextPage=!1);let o=s.querySelector("a")?.href??"";c.nextPageLink=o,c.nextPage=p(o)??0;break}case s.classList.contains("prev"):{let o=s.querySelector("a")?.href??"";c.previousPageLink=o,c.previousPage=p(o)??void 0;break}default:{let o=c.total,m=parseFloat(s.querySelector("a")?.textContent??"0");o===void 0&&(c.total=m),Number(o)<m&&(c.total=m)}}return{type:i.List,metadata:c,count:r.length,torrents:r.filter(s=>Object.values(s).every(o=>o!==void 0))}},N,w).finally(async()=>await this.job.finishedTask())}async details(e,t){let i=new URL(`/view/${e}`,this.url).toString();return await this.waitForConnect(i),await this.job.api.evaluate((a,n,u,l)=>{let r={type:u.Details,description:void 0,submitter:void 0,information:void 0,files:void 0,comments:void 0};if(n===!0||n!==!1&&n?.submitter){let p=document.querySelector('[title="User"]'),f=p?.textContent?.trim(),s=p?.getAttribute("href");f&&s&&(r.submitter={name:f,url:new URL(s,a).toString()})}if(n===!0||n!==!1&&n?.information){let f=document.querySelector("body > div > div:nth-child(1) > div.panel-body > div:nth-child(3) > div:nth-child(2)")?.textContent?.trim();r.information=f}if(n===!0||n!==!1&&n?.description){let p=document.getElementById("torrent-description");switch(n){case!0:{r.description=p?.outerHTML??p?.getHTML();break}default:switch(n.description){case"text":{r.description=p?.textContent??p?.innerText??"";break}default:r.description=p?.outerHTML??p?.getHTML()}}}if(n===!0||n!==!1&&n?.files){let p=function(o){let m={B:1,KiB:1024,MiB:1048576,GiB:1073741824,TiB:1099511627776,PiB:0x4000000000000},b=/([\d.]+)\s*(B|KiB|MiB|GiB|TiB|PiB)/i,g=o.match(b);if(!g)throw new Error(`Invalid size format: ${o}`);let[,x,y]=g,T=parseFloat(x);if(!Object.prototype.hasOwnProperty.call(m,y))throw new Error(`Unknown unit: ${y}`);let D=m[y];return T*D},f=function(o){let m=[],b=Array.from(o.querySelectorAll(":scope > ul > li"));for(let g of b){let x=g.querySelector(":scope > a.folder");if(x){let y=x.textContent?.trim();if(!y)return m;let T=f(g);m.push({type:l.Folder,name:y,files:T})}else{let y=g.querySelector(":scope > span.file-size")?.textContent?.trim();if(!y)return m;let T=g.textContent?.replaceAll(y,"");if(!T)return m;let D=y.replace("(","").replace(")","");m.push({type:l.File,fileName:T,readableSize:D,sizeInBytes:p(D)})}}return m};var d=p,c=f;let s=document.querySelector(".torrent-file-list");if(s){let o=f(s);r.files=o}}if(n===!0||n!==!1&&n?.comments){let p=document.querySelectorAll(".comment-panel"),f=[];for(let s of Array.from(p)){let o=s.querySelector("img")?.src??"",m=s.querySelector('[title="User"]')?.textContent?.trim()??"",b=s.querySelector('[title="User"]')?.parentElement?.textContent?.trim().includes("uploader")??!1,g=s.querySelector("[markdown-text]")?.textContent?.trim()??"",x=s.querySelector("[data-timestamp-swap]"),y=x?.getAttribute("data-timestamp"),T=x?.getAttribute("title");!o||!m||!g||!y||!T||f.push({avatarURL:o,userName:m,message:g,publishDate:T,timestamp:parseInt(y),isUploader:b})}f.length>0&&(r.comments=f)}return r},this.url,t,w,C).then(a=>t===!0||t!==!1&&t?.description===!0||t!==!1&&!["html","text"].includes(t?.description??"")?{...a,description:_.translate(a.description??"")}:a).finally(async()=>await this.job.finishedTask())}async waitForConnect(e){let t;do{t&&await new Promise(a=>setTimeout(()=>a(),this.job.timeout));let i=await this.job.api.goto(e,{waitUntil:"domcontentloaded"});i?.status()===429&&console.log("The site detected too many hits: 429 Too Many Requests"),t=i?.status()}while(t===429)}};var J=class h{pagesToLoad=1;loadAdditionalInfo;client;constructor(e){this.client=e.client,e?.pagesToLoad&&(this.pagesToLoad=e.pagesToLoad),this.loadAdditionalInfo=e?.loadAdditionalInfo??!1}async search(e,t,i){let a=Date.now(),n=(await Promise.all(Array.from({length:t?.loadOnlyPage&&t?.page?1:this.pagesToLoad},async(d,c)=>{let p=t?.loadOnlyPage&&t?.page?t.page:(t?.page?t.page-1:0)+(c+1),f=await this.client.requestJob(),s=h.getSearchParams(e,p,t?.filter);switch(this.client.runTime){case"puppeteer":return new v({job:f,url:this.client.getURL(),loadAdditionalInfo:this.loadAdditionalInfo}).extract(s);case"cheerio":return new R({job:f,loadAdditionalInfo:this.loadAdditionalInfo,url:this.client.getURL()}).extract(s)}}))).filter(d=>d!==void 0),u=n.reduce((d,c)=>d.concat(c.torrents),i?.getData()?.torrents??[]);this.loadAdditionalInfo!==!1&&await Promise.all(Array.from({length:n.length},async(d,c)=>{let p=n[c];await Promise.all(Array.from({length:p.torrents.length},async(f,s)=>{let o=p.torrents[s],m=await this.details(o.id,this.loadAdditionalInfo);o.details=m}))}));let l=n.reduce((d,c)=>c.metadata.current>d.metadata.current?c:d,{metadata:{current:-1/0}}),r=Date.now();return new O({scraper:this,search:{content:e,filter:t?.filter},data:{type:"list",metadata:{...l.metadata,timeTaken:(r-a)/1e3},count:u.length,torrents:u}})}async details(e,t){let i=h.normalizeId(e);if(!i)throw new Error(`Invalid torrent identifier: ${e}`);e=i;let a=await this.client.requestJob(),n=new URL(`/view/${e}`,this.client.getURL()).toString();switch(this.client.runTime){case"puppeteer":return await new v({url:this.client.getURL(),job:a,loadAdditionalInfo:this.loadAdditionalInfo}).details(n,t);default:return await new R({url:this.client.getURL(),job:a,loadAdditionalInfo:this.loadAdditionalInfo}).details(n,t)}}static getSearchParams(e,t=1,i){let a=String(i?.filter??0),n=i?.category?.split("."),u="0_0";if(n){let[l,r]=n,d=Object.entries(B).findIndex(([p])=>p===l),c=Object.entries(B)[d][1].findIndex(p=>r===p);u=`${d+1}_${c+1}`}return new URLSearchParams({f:a,c:u,q:e,p:String(t)}).toString()}static normalizeId(e){if(typeof e=="number")return e;if(typeof e=="string"){let t=parseInt(e,10);return isNaN(t)?h.extractId(e)??null:t}return typeof e=="object"&&"id"in e&&typeof e.id=="number"?e.id:null}static extractId(e){let t=/^https:\/\/nyaa\.si\/view\/(\d+)$/,i=e.match(t)?.[1];return i?parseInt(i):void 0}};export{R as Cheerio,I as Client,w as DataTypes,N as ElementPropertyTypes,O as Exporter,C as FileEntityType,B as FilterObject,H as Filters,P as Job,F as JobStatus,A as Method,v as Puppeteer,E as RunTimes,J as Scraper,k as SessionManager,L as TypedEventEmitter};