tiktok-scraper-ts-updated/dist/main.js

A scraper for tiktok videos, users and more

"use strict";var P=Object.create;var v=Object.defineProperty;var W=Object.getOwnPropertyDescriptor;var F=Object.getOwnPropertyNames;var J=Object.getPrototypeOf,Y=Object.prototype.hasOwnProperty;var a=(o,r)=>v(o,"name",{value:r,configurable:!0});var z=(o,r)=>{for(var t in r)v(o,t,{get:r[t],enumerable:!0})},M=(o,r,t,e)=>{if(r&&typeof r=="object"||typeof r=="function")for(let n of F(r))!Y.call(o,n)&&n!==t&&v(o,n,{get:()=>r[n],enumerable:!(e=W(r,n))||e.enumerable});return o};var p=(o,r,t)=>(t=o!=null?P(J(o)):{},M(r||!o||!o.__esModule?v(t,"default",{value:o,enumerable:!0}):t,o)),G=o=>M(v({},"__esModule",{value:!0}),o);var ot={};z(ot,{Music:()=>b,TTScraper:()=>l,TikTokResult:()=>I,User:()=>f,Video:()=>g,fetchAllVideosFromUser:()=>tt,fetchMusic:()=>et,fetchUser:()=>Z,fetchVideo:()=>Q,fetchVideoNoWaterMark:()=>rt,hashtag:()=>it});module.exports=G(ot);var S=p(require("cheerio")),T=p(require("miniget")),U=p(require("node-fetch")),h=require("fs"),R=p(require("puppeteer")),L=p(require("http")),V=p(require("https")),O=require("process");var f=class{constructor(r,t,e,n,i,u,d,c,s,w,_,k,A,C){this.id=r,this.uniqueId=t,this.nickname=e,this.avatar=n,this.signature=i,this.createdAt=u,this.verified=d,this.secretUID=c,this.bioLink=s,this.privateAccount=w,this.followers=_,this.following=k,this.hearts=A,this.videos=C}};a(f,"User");var I=class{constructor(r,t,e,n,i,u,d,c,s,w){this.author=r,this.video=t,this.audio=e,this.shareCount=n,this.likesCount=i,this.commentCount=u,this.playCount=d,this.createdAt=c,this.tiktokLink=s,this.thumbnail=w}};a(I,"TikTokResult");var g=class{constructor(r,t,e,n,i,u,d,c,s,w,_,k,A,C,$,D,j,q){this.id=r,this.description=t,this.createdAt=e,this.height=n,this.width=i,this.duration=u,this.resolution=d,this.shareCount=c,this.likesCount=s,this.commentCount=w,this.playCount=_,this.downloadURL=k,this.cover=A,this.dynamicCover=C,this.playURL=$,this.format=D,this.author=j,this.directVideoUrl=q}};a(g,"Video");var b=class{constructor(r,t,e,n,i,u,d,c,s){this.id=r,this.title=t,this.playURL=e,this.coverLarge=n,this.coverThumb=i,this.author=u,this.duration=d,this.original=c,this.album=s}};a(b,"Music");var N=require("netscape-cookies-parser");var E=p(require("node-fetch"));var H=require("tiktok-signature"),K="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56",B="https://www.tiktok.com/api/post/item_list/?WebIdLastTime=1725387734&aid=1988&app_language=en&app_name=tiktok_web&browser_language=en-US&browser_name=Mozilla&browser_online=true&browser_platform=MacIntel&browser_version=5.0 (Macintosh)&channel=tiktok_web&cookie_enabled=true&count=35&coverFormat=2&cursor=0&data_collection_enabled=true&device_id=7410483808526698016&device_platform=web_pc&focus_state=true&from_page=user&history_len=5&is_fullscreen=false&is_page_visible=true&language=en&odinId=7410483842936669216&os=mac&priority_region=&referer=https://www.google.com/&region=PL&root_referer=https://www.google.com/&screen_height=1117&screen_width=1728&secUid=MS4wLjABAAAAGu-Kg2l1tIS7B6HJ1skdamgWbjdobxERVTR2NJ0MHXg0X7E8b_Pt1E7KUp8-bSzc&tz_name=Europe/Warsaw&user_is_login=false&webcast_language=en&msToken=4plSj1K-DnpCGX4S4YB8UcoEs6Jh-jrEyjH9Ot8W0CRZUk0gSkPxIzhSLctxAGwlwMwKbVNFYO7fO6Eze0YIqYjK57eT2_wxETNqXjU197SxS3zSOx7YXkc6iGULBc0vd1vzsrPVMB5S3zM-2eF3jg==&X-Bogus=DFSzsIVOSz2AN9fbtI/AvSMK1-7s&_signature=_02B4Z6wo00001k5X6aQAAIDCNwU9tBpuJepOVe0AAPVReb",y={aid:"1988",count:35,secUid:"",cursor:"",cookie_enabled:!0,screen_width:0,screen_height:0,browser_language:"",browser_platform:"",browser_name:"",browser_version:"",browser_online:"",timezone_name:"Europe/London"};async function x(o,r){y.secUid=o,y.cursor=r;let t=new H(null,K);await t.init();let i=`https://m.tiktok.com/api/post/item_list/?${new URLSearchParams(y).toString()}`,u=await t.sign(i),d=await t.navigator();await t.close();let{"x-tt-params":c}=u,{user_agent:s}=d,w=await X(s,c);if(w.status!==200)throw new Error("A request to get the user's videos was not successful!");return await w.json()}a(x,"getUserVideos");async function X(o,r){return await(0,E.default)(B,{method:"GET",headers:{"user-agent":o,"x-tt-params":r}})}a(X,"fetchVideos");var l=class{constructor(r){this._cookiesJar=new N.CookieJar;r?this._cookies=this._cookiesJar.load(r):this._cookies=""}async requestWebsite(r,t){let e=new L.default.Agent({keepAlive:!0,maxSockets:20}),n=new V.default.Agent({keepAlive:!0,maxSockets:20}),i={agent:s=>s.protocol=="http:"?e:n,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36",Accept:"application/json, text/plain, */*","Accept-Encoding":"gzip, deflate, br","Cache-Control":"no-cache",Connection:"keep-alive",Cookie:`${this._cookies?this._cookiesJar.toString():""}`}},d=await(await(0,U.default)(`${r}`,t||i)).text();return S.load(d,{xmlMode:!0})}extractJSONObject(r){let t=r.split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1].indexOf("<\/script>");return r.split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1].slice(0,t)}checkJSONExisting(r){try{return!!JSON.parse(r)}catch{}}async requestWithPuppeteer(r){let t=await R.launch({headless:!0,args:["--no-sandbox","--disable-setuid-sandbox"]}),n=await(await t.newPage()).goto(r,{waitUntil:"domcontentloaded"});if(n==null)throw await t.close(),new Error("Could not load the desired Page!");let i=await n.text();return await t.close(),this.extractJSONObject(i)}handleHTMLContent(r){let t=r,e=t.split("window['SIGI_STATE']=")[1].indexOf(";window['SIGI_RETRY']=");return JSON.parse(t.split("window['SIGI_STATE']=")[1].slice(0,e))}async TryFetch(r){let t=await this.requestWebsite(r);if(this.checkJSONExisting(t("#__UNIVERSAL_DATA_FOR_REHYDRATION__").text()))return JSON.parse(t("#__UNIVERSAL_DATA_FOR_REHYDRATION__").text());{let e=await this.requestWithPuppeteer(r);return JSON.parse(e)}}async video(r){if(!r)throw new Error("A video URL must be provided");let e=(await this.TryFetch(r)).__DEFAULT_SCOPE__["webapp.video-detail"].itemInfo.itemStruct,n=e.id;if(n==0)return console.log("Could not find the Video on Tiktok!");let i=await this.noWaterMark(r);return new g(n,e.desc,new Date(Number(e.createTime)*1e3).toLocaleDateString(),Number(e.video.height),Number(e.video.width),Number(e.video.duration),e.video.ratio,e.stats.shareCount,e.stats.diggCount,e.stats.commentCount,e.stats.playCount,i,e.video.cover,e.video.dynamicCover,i,e.video.format,e.author,`https://www.tiktok.com/@${e.author.uniqueId}/video/${n}`)}async user(r){if(!r)throw new Error("Please enter a username");let e=(await this.TryFetch(`https://www.tiktok.com/@${r}`)).__DEFAULT_SCOPE__["webapp.user-detail"].userInfo;return new f(e.user.id,e.user.uniqueId,e.user.nickname,e.user.avatarLarger,e.user.signature.trim(),new Date(e.createTime*1e3).toLocaleDateString(),e.user.verified,e.user.secUid,e?.bioLink?.link??"none",e.user.privateAccount,e.stats.followerCount,e.stats.followingCount,e.stats.heart,e.stats.videoCount)}async getAllVideosFromUser(r,t){if(!r)throw new Error("You must provide a username!");let{secretUID:e}=await this.user(`${r}`);if(!e)throw new Error("Couuld not find user UID!");let n="",i=[],u=[],d=await x(e,n);if(d?.itemList&&(u.push(d.itemList),n=d.cursor),d?.hasMore===!0)for(;;){let c=await x(e,n);if(u.push(c.itemList),n=c.cursor,c.hasMore==!1)break}for(let c of u)for(let s of c)i.push(new g(s.id,s.desc,new Date(Number(s.createTime)*1e3).toLocaleDateString(),Number(s.video?.height),Number(s.video?.width),Number(s.video?.duration),s.video?.ratio,s?.stats?.shareCount,s?.stats?.diggCount,s?.stats?.commentCount,s?.stats?.playCount,t?await this.noWaterMark(`https://www.tiktok.com/@${s.author.uniqueId}/video/${s.id}`):s.video?.downloadAddr.trim(),s?.video?.cover,s?.video?.dynamicCover,t?await this.noWaterMark(`https://www.tiktok.com/@${s.author.uniqueId}/video/${s.id}`):s.video?.downloadAddr.trim(),s?.video?.format,s.author,`https://www.tiktok.com/@${s.author.uniqueId}/video/${s.id}`));return i}async getMusic(r){if(!r)throw new Error("You must provide a link!");let e=(await this.TryFetch(r)).__DEFAULT_SCOPE__["webapp.video-detail"].itemInfo.itemStruct;return new b(e.music.id,e.music.title,e.music.playUrl,e.music.coverLarge,e.music.coverThumb,e.music.authorName,Number(e.music.duration),e.music.original,e.music.album)}async downloadAllVideosFromUser(r,t){if(!r)throw new Error("Please enter a username!");let e=await this.getAllVideosFromUser(r);if(!e)throw new Error("No Videos were found for this username. Either the videos are private or the user has not videos");if(!t.path){if(t.path=`${__dirname}/../${r}`,(0,h.existsSync)(t.path)){console.log("A folder with this username exists, that is unusual!");try{(0,h.unlinkSync)(t.path)}catch(n){console.log(`[ERROR] Could not remove ${t.path}
 Error Message: ${n.message}`),(0,O.exit)(1)}}(0,h.existsSync)(t.path)||(0,h.mkdirSync)(t.path)}if(t.watermark){for(let[n,i]of e.entries()){console.log(`Downloading Video: ${i.description?i.description:i.id}, [${n+1}/${e.length}]`);let u=await this.noWaterMark(i.id);if(!u){console.log(`Could not fetch ${i.description?i.description:i.id} with no watermark`);continue}(0,T.default)(u).pipe((0,h.createWriteStream)(`${t.path}/${i.id}_${i.resolution}.${i.format}`))}return}for(let[n,i]of e.entries())console.log(`Downloading Video: ${i.description?i.description:i.id}, [${n+1}/${e.length}]`),(0,T.default)(i.downloadURL).pipe((0,h.createWriteStream)(`${t.path}/${i.id}_${i.resolution}.${i.format}`))}async noWaterMark(r){let t={url:r,count:"12",cursor:"0",web:"1",hd:"1"},e=await(0,U.default)("https://www.tikwm.com/api/",{method:"POST",headers:{"Content-Type":"application/x-www-form-urlencoded"},body:new URLSearchParams(t).toString()});if(!e.ok)throw new Error("There was an Error retrieveing this video without watermark!");let n=await e.json();if(n.code===-1)throw new Error("API Limit for nowatermark, please wait 1 second and try again!");return"https://www.tikwm.com"+n.data.hdplay}async hashTag(r){if(!r)throw new Error("You must provide a tag name to complete the search!");let t=await this.TryFetch(`https://www.tiktok.com/tag/${r}`);console.log(t);let{ItemList:e}=t,n=[];for(let i of e.challenge.list)n.push(new g(t.ItemModule[i].video.id,t.ItemModule[i].desc,new Date(Number(t.ItemModule[i].createTime)*1e3).toLocaleDateString(),Number(t.ItemModule[i].video.height),Number(t.ItemModule[i].video.width),Number(t.ItemModule[i].video.duration),t.ItemModule[i].video.ratio,t.ItemModule[i].stats.shareCount,t.ItemModule[i].stats.diggCount,t.ItemModule[i].stats.commentCount,t.ItemModule[i].stats.playCount,t.ItemModule[i].video.downloadAddr.trim(),t.ItemModule[i].video.cover,t.ItemModule[i].video.dynamicCover,t.ItemModule[i].video.playAddr.trim(),t.ItemModule[i].video.format,t.ItemModule[i].author));return n}};a(l,"TTScraper");async function Q(o){if(!o)throw new Error("You must provide a Tiktok video url!");return await new l().video(o)}a(Q,"fetchVideo");async function Z(o){if(!o)throw new Error("You must provide a username!");return await new l().user(o)}a(Z,"fetchUser");async function tt(o,r){if(!o)throw new Error("You must provide a username!");return await new l().getAllVideosFromUser(o,r)}a(tt,"fetchAllVideosFromUser");async function et(o){if(!o)throw new Error("You must provide a Tiktok video url!");return await new l().getMusic(o)}a(et,"fetchMusic");async function rt(o){if(!o)throw new Error("You must provide a Tiktok video url!");return await new l().noWaterMark(o)}a(rt,"fetchVideoNoWaterMark");async function it(o){if(!o)throw new Error("You must provide a tiktok hashtag");return await new l().hashTag(o)}a(it,"hashtag");0&&(module.exports={Music,TTScraper,TikTokResult,User,Video,fetchAllVideosFromUser,fetchMusic,fetchUser,fetchVideo,fetchVideoNoWaterMark,hashtag});