@davidpunya/web-scraper
Version:
The library web scraper for Restfull API's
572 lines (562 loc) • 26.2 kB
JavaScript
const axios = require("axios");
const FormData = require("form-data");
const cheerio = require("cheerio");
const fs = require("fs");
const util = require("util");
const Function = require("../dist/functions");
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
const ytdl = require("ytdl-core");
const yts = require("yt-search");
const Func = new Function();
module.exports = class Downloader {
fb = async (link) => {
const form = new FormData();
form.append("URLz", link);
const result = {
status: 200,
creator: "David XD",
result: {
videoInfo: {},
video: {}
}
};
await axios({
url: "https://fdown.net/download.php",
method: "POST",
headers: {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Content-Type": "application/x-www-form-urlencoded",
"Cookie": "_ga=GA1.1.641272189.1717930861; __gads=ID=b7d483cd12951c89:T=1717930890:RT=1718965557:S=ALNI_MbmcxUNBQsQWeICXRn75L4Y3oDsFg; __gpi=UID=00000e44fcf1acad:T=1717930890:RT=1718965557:S=ALNI_MYxWOBXikbZuJ3DnIJnF_ySvU9nGw; __eoi=ID=cd15c7096be29492:T=1717930890:RT=1718965557:S=AA-AfjYBSwCwKv03QYc2Z6Mk4xXA; FCNEC=%5B%5B%22AKsRol-n0Pn4Lwdclbzlve3oZ8XkN7sEPHn2aoJR-H8_Z3OAmVcAwPdUistkUHhH5xFCgN_NzSRtfXn6kshqUii4juKUbxLK8QqOmqEU7QsRmu3-a54PIoEHNfSUzbOC8O3Ot4hBnkW23QyAHqoIA3AdBJnNOZihhA%3D%3D%22%5D%5D; cf_clearance=5AFjKVncgA58kfe4PRKV4kdDJKrHKlKY1gu6kkKcuuM-1718965579-1.0.1.1-bAVMyEGGwBzGf92LJfJ.eSPR4aKYU95GIuM2H2G66H090MEQfGPfsLNNwGWAJjDMMJu3K3GlZQf3b8UtWrgGXw; _ga_82ERN9JZD3=GS1.1.1718965514.2.1.1718965600.57.0.0",
"User-Agent": "GoogleBot"
},
data: form
}).then(({ data }) => {
const $ = cheerio.load(data);
$("#result > .col-xs-6.col-xs-offset-3.no-padding.lib-item > .lib-panel").each(function() {
result.result.videoInfo = {
title: $(this).find(".col-md-6").eq(1).find(".lib-row.lib-desc").eq(0).text().trim().split("Description: ").join(""),
duration: $(this).find(".col-md-6").eq(1).find(".lib-row.lib-desc").eq(1).text().trim().split("Duration: ").join(""),
thumb: $(this).find(".col-md-6").eq(0).find("img").attr("src")
}
});
$("#result > .col-xs-12").each(function() {
let hd = $(this).find(".col-md-4 > a").attr("href");
let sd = $(this).find("#sd > a").attr("href");
result.result.video = {
hd: hd ? hd : "Video With HD Resolution Not Found :/",
sd: sd ? sd : "Video With SD Resolution Not Found :/"
};
});
}).catch((er) => {
console.error(er);
return JSON.stringify({
status: 500,
creator: "David XD",
msg: util.format(er)
});
});
//console.log(JSON.stringify(result, null, 2));
return result;
};
igdl = async(link) => {
try {
const data = await axios("https://fastdl.app/api/convert", {
method: "POST",
headers: {
"User-Agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
"Accept": "application/json, text/plain, */*",
"Content-Type": "application/json"
},
data: {
"url": link,
"ts": 1717886361080,
"_ts": 1717498039111,
"_tsc": 2178064,
"_s": "881325deb3a090678823ebc67026858605bca3f91df3f3b96e0eaac7965a9754"
}
})
let result = {
status: 200,
creator: "David XD",
result: data.data
}
return result;
} catch (er) {
console.error(er)
}
}
pinterestdl = async(link) => {
const form = new FormData();
form.append("url", link);
const result = {
status: 200,
creator: "David XD",
data: {
url: link,
result: ""
}
};
await axios({
url: "https://pinterestvideodownloader.com/download.php",
method: "POST",
headers: {
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Content-Type": "application/x-www-form-urlencoded",
"Cookie": "_ga_966QNV4G77=GS1.1.1718265709.1.1.1718265710.0.0.0; _ga=GA1.2.431955486.1718265710; _gid=GA1.2.1691914427.1718265710; __gads=ID=a768755ea54ad065:T=1718265744:RT=1718265744:S=ALNI_MYhy1D7j7Sk-L38lY0gCrvHslkj9w; __gpi=UID=00000e4a44effcb5:T=1718265744:RT=1718265744:S=ALNI_MYlyVI3dB_rxdfiktijz5hqjdFh3A; __eoi=ID=bcaa659e3f755205:T=1718265744:RT=1718265744:S=AA-AfjaNqVe1HORKDn3EorxJl5TE; FCNEC=%5B%5B%22AKsRol-DFkw9G-FS4szSzz5S-Zy-awxxF02UE3axThxkDqbMdR-KD0ss2AkukIaNNXn-fXts6XPmkNEPhKLEh-MWatFyvpof-XZuWVyQDQIAatU_iGwEIPl3TYlsnsZdyNvsNGsr0w0yz2xNc-o7rSwnGm5sWti7ag%3D%3D%22%5D%5D",
"Origin": "https://pinterestvideodownloader.com",
"Referer": "https://pinterestvideodownloader.com/id/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
},
data: form
}).then(( { data }) => {
const $ = cheerio.load(data);
let _$ = $("div.wrapper.clearfix > div#main.maxwidth.clearfix > main#content.content > center").eq(0);
result.data.result = _$.find("div.col-sm-12 > video").attr("src");
}).catch((e) => {
console.error(e);
return JSON.stringify({
status: 500,
creator: "David XD",
msg: util.format(er)
});
});
//console.log(result);
return result;
}
twitterdl = async(link) => {
const form = new FormData();
form.append("gsc", "03AFcWeA4dMAaByNmUsoBZverpVnItnPaOO5cK_bKzqIKAAuBcltIh2W2CI9WGAoAVSDa3nKT3EcBCyeS1X6SA0LWSg8r_7xuTi2050H5MqbTjVKDuZmXEsZFSgBuYkg7cjh_btzBsnW7JTt5l9b5NwpjoP3BfG4Clf4CIAd2HA4XxRoM8wjq2WT0d-eI4LuklGoY2LH8HZnkQbRFu0WBBGHz5LH0bIqqxCtR0cv258z2KtUd8JPVWQdl7_NNNG_fAzJRMwzr9_juAf1NphZLNkXE55WLPiXd7165_KuC5Jy32ceEEsttZR1HmsREbBqTSqIEbIoLFIh1rJj1V2J92XHphfLBK3vTSpkpHq_KI-zec1DPczFKbTKOPZkEcQk0K94VrQzUO_4atAyf1W1Xfp2HluQZhX_GTeFTWdxtbfuOWSgpENyXP7lMEF2ryaQ5eGI0pPX3JnC9Wg7pflhTZy1fP_gHBI7kiSbF5OCclEgRQw6Gr83M9uZLTGkLvevsPB4Ltbq_vIhQruO52F2z_Rqp_8x-eV2MfhiZuJhn1E8dT40ptpV7ch6J7KT7I0C0Qc6HHPgwLX67OpeNCuhM3XIIBmfuablKHb0iWXrymXHyHQzSl4pAwa8PRY9tp1SeERefpHHbKGwv9O8EJWgy8PswE5jUvHK5Ref2gtJzpo8HWjS4VURtda-m4RG3fHFBRVzCkGrECw6va_EuHeXlx4jtMfk2UUTB-HCFfXMdlH1IVPg7PwHVOqIZLiWfv5Fw7Zz0V9aVFVAfrP5NtgHmxTseoPGqhL-YTxV7oaUDZsUa7iQypzJUIz3_Y1delIdyANH1TcRAaAVeJby6im_YH-tNPK29FfhIGfTFFmzqSuSMHWCSgNhwn_um6Csf4bHDu_g56_Q9QEjYBRZdhm8g3jB_W8bQPGjyfBXOuNY1B5D8pkwecdO6wqGHFgv-vejRRZuhF05Rvyh6NWeCNWXggEfwuSnE5nhCn_DRE0P739nOEjjUoxA-M2_YZsRsXaPHdT_8jrUWNY5WatNh9S612cVBJa2UIh80JCFJ19OfB_8e-16jfAo-xVGxO50a1l6HVdyGo5VOM1TDk-kz23Iu58auEy_V4nhrTiWevh_IxHY6xgyw6P6FGhfYpLgWe-kM8TFg5zswvkIo5_VVNIamJytxwLRYJewyhMT6cS-E8rf2DQZ2BPUZTLR2Qny2c1BE4xpgXOCd0l_Ip1y4Ox3GEu297n0eefh6kVpmqvVJ_fVd11ecd6ESuMC6lj9KhlCDI7FJrT8Bc_9ss1cCo6H94zwkHrcWLGYB-Ec-QqevAIwbOQ50VZ61pLPrZQxBWPGteGANH07Qrff42bC6xKwvrMv7MNna1uw_uh_1vG2h6QKBETAtRTIqSOAnEXYN85US00B4GAr8NfmOt45GHtp1L2zO3KAjZ_SZtcMj84emqWXL5rjrWOTwqFSBkTxHQOrRUhw3Rvj762hXXwYpafsK1XRYb9AO0OBya_6wU20x4uv61yYEH9lcSqv_HHWcCjKvk59xVvS7nUiwOjBHVpJIP26omG5V9_ER-FFROfHaDqzk0cF6Qp3J186t8oMlzNlJ-jUySzq97_yg8LPsgV4H8r11Zq0UmSbwusspxQ2lODSUH97CQg8YX7hyMA4FLOUqExJiIErAALl_xBo0w-nolssYQvQV_7Vx68e4PqlMM32n5q62DeKBg8OTsA7LuZTLIuaxC_EGzsytE");
form.append("target", link);
const result = {
status: 200,
creator: "David XD",
result: {}
};
try {
const res = await (await fetch("https://api.snapfirecdn.com/twitter", {
method: "POST",
headers: {
"Accept": "application/json",
"Content-Type": "application/json",
"Origin": "https://snaptwitter.io",
"Referer": "https://snaptwitter.io/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
},
body: form
}));
let tipe = res.medias.find(v => v.type);
let data;
if (tipe === "photo") {
data = res.medias.map(p => ({
content_type: p.content_type,
media_url: p.media,
thumb: p.thumbnail,
type: p.type
}));
} else {
data.push = {
content_type: res.medias[0].content_type,
media_url: res.medias[0].media,
thumb: res.medias[0].thumbnail,
type: res.medias[0].type
};
};
result.result = {
author: res.username,
src: data
};
} catch (e) {
console.error(e);
return JSON.stringify({
status: 500,
creator: "David XD",
msg: util.format(e)
});
};
//console.log(result);
return result;
};
spotifydl = async (link) => {
try {
const res = await (await axios("https://api.fabdl.com/spotify/get?url=" + link, {
method: "GET",
headers: {
Accept: "application/json, text/plain, */*",
"Origin": "https://keepvid.ch",
"Referer": "https://keepvid.ch/spotify-download-new12",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
}
})).data.result;
let id = res.id;
let gid = res.gid;
let aud = await (await axios.get("https://api.fabdl.com/spotify/mp3-convert-task/" + gid + "/" + id)).data.result;
let result = {
status: 200,
creator: "David XD",
result: {
title: res.name,
author: res.artists,
type: res.type,
duration: res.duration_ms,
thumb: res.image,
play: "https://api.fabdl.com" + aud.download_url,
ext: ".mp3",
id: id,
gid: gid
}
};
//console.log(JSON.stringify(result, null, 2));
return result;
} catch (error) {
console.error(error);
return JSON.stringify({
status: 500,
creator: "David XD",
msg: error.message
});
}
};
tiktokdl = async(url) => {
try {
const res = await (await axios("https://www.tikwm.com/api/", {
method: "POST",
headers: {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Origin': 'https://www.tikwm.com',
'Referer': 'https://www.tikwm.com/',
'Sec-Ch-Ua': '"Not)A;Brand" ;v="24" , "Chromium" ;v="116"',
'Sec-Ch-Ua-Mobile': '?1',
'Sec-Ch-Ua-Platform': 'Android',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
},
params: {
url: url,
count: 12,
cursor: 0,
web: 1,
hd: 1
}
})).data.data
let src = []
if (!res.size) {
res.images.map(puki => {
src.push({
type: "image",
url: puki
})
})
} else {
src.push({
type: "nowm",
url: "https://www.tikwm.com" + res.play,
size: Func.bytesToSize(res.size)
}, {
type: "wm",
url: "https://www.tikwm.com" + res.wmplay,
size: Func.bytesToSize(res.wm_size)
}, {
type: "nowm_hd",
url: "https://www.tikwm.com" + res.hdplay,
size: Func.bytesToSize(res.hd_size)
})
}
if (res.duration === 0) {
var dur = "This Is Image"
}
let result = {
status: true,
creator: "David XD",
result: {
videoInfo: {
id: res.id,
title: res.title,
region: res.region,
video_cover: "https://www.tikwm.com" + res.cover,
duration: res.duration ? res.duration : dur,
created_at: Func.formatDate(res.create_time * 1000),
data: src,
count: {
play: Func.formatCount(res.play_count),
like: Func.formatCount(res.digg_count),
comment: Func.formatCount(res.comment_count),
save: Func.formatCount(res.collect_count),
share: Func.formatCount(res.share_count),
download: Func.formatCount(res.download_count),
}
},
author: {
id: res.author.id,
tag: "@" + res.author.unique_id,
name: res.author.nickname,
avatar: "https://www.tikwm.com" + res.author.avatar
},
musicInfo: {
id: res.music_info.id,
title: res.music_info.title,
author: res.music_info.author,
original: res.music_info.original,
duration: Func.formatSeconds(res.music_info.duration),
album: res.music_info.album ? res.music_info.album : "Unknown",
url: res.music_info.play
}
}
}
//console.log(result);
return result;
} catch(e) {
console.error(e)
return {
status: 500,
creator: "David XD",
msg: util.format(e)
}
}
};
mediafire = async(link) => {
if (!/^https?:\/\/(www\.)?mediafire\.com\/.+/.test(link)) return {
status: 400,
creator: "David XD",
msg: "Invalid Link!!!"
};
const result = {
status: 200,
creator: "David XD",
result: {}
};
await axios({
url: link,
method: "GET"
}).then((res) => {
const $ = cheerio.load(res.data);
const file = $(".content > .center > .ads.dl-box > .dl-info > .intro > .filename").text().trim();
result.result = {
filename: file,
size: $(".content > .center > .ads.dl-box > .dl-info > .details > li").eq(0).find("span").text().trim(),
upload_at: $(".content > .center > .ads.dl-box > .dl-info > .details > li").eq(1).find("span").text().trim(),
mime_type: file.split('.').pop(),
url: link,
media_url: $("#download_link > #downloadButton").attr("href")
};
}).catch((err) => {
console.error(err);
return {
status: 500,
creator: "David XD",
msg: util.format(err)
};
});
//console.log(result);
return result;
};
ytmp3 = async(url) => {
const regex = /^(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:watch\?v=|shorts\/)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
if (!regex.test(url)) {
return {
status: 404,
creator: "David XD",
msg: "Invalid URL!!!"
};
} else {
try {
const a = (await ytdl.getInfo(url, {
lang: "id"
})).videoDetails;
const b = ytdl(url, {
filter: "audioonly",
quality: 140
});
const c = [];
b.on("data", (cc) => {
c.push(cc);
});
await new Promise((resolve, reject) => {
b.on("end", resolve);
b.on("error", reject);
});
const d = Buffer.concat(c);
let e = a.uploadDate ?? "Unknown";
let f = new Date(e).toLocaleString();
return {
status: 200,
creator: "David XD",
data: {
videoInfo: {
title: a.title ?? "Unknown",
views: a.viewCount ?? "Unknown",
duration: Func.formatSeconds(a.lengthSeconds) ?? "Unknown",
upload_at: f ?? "Unknown",
video_id: a.videoId ?? "Unknown",
video_url: a.video_url ?? "Unknown",
thumb: (a.thumbnails.slice(-1)[0]?.url) ?? "Unknown",
desc: a.description ?? "Unknown"
},
authorInfo: {
channel: {
name: a.author.name ?? "Unknown",
user_tag: a.author.user ?? "Unknown",
id: a.author.id ?? "Unknown",
subs: a.author.subscriber_count ?? "Unknown",
url: a.author.channel_url ?? "Unknown",
url_tag: a.author.user_url ?? "Unknown",
avatar: a.author.thumbnails[2].url ?? "Unknown"
}
},
audioInfo: {
audio_buffer: d,
size: Func.bytesToSize(d.length)
}
}
};
} catch (error) {
console.error(error);
return {
status: 500,
creator: "David XD",
msg: util.format(error)
};
}
};
};
ytmp4 = async(url, quality = "highestvideo") => {
const regex = /^(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:watch\?v=|shorts\/)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
if (!regex.test(url)) {
return {
status: 404,
creator: "David XD",
msg: "Invalid URL!!!"
};
} else {
try {
const a = await ytdl.getInfo(url, {
lang: "id"
});
const b = ytdl.chooseFormat(a.formats, {
quality: quality,
filter: "videoandaudio"
});
const c = (await fetch(b.url, {
method: "HEAD"
})).headers.get("content-length")
const cc = parseInt(c);
const d = a.videoDetails;
let e = d.uploadDate ?? "Unknown";
let f = new Date(e).toLocaleString();
return {
status: 200,
creator: "David XD",
data: {
videoInfo: {
title: d.title ?? "Unknown",
views: d.viewCount ?? "Unknown",
duration: Func.formatSeconds(d.lengthSeconds) ?? "Unknown",
quality: b.qualityLabel ?? "Unknown",
upload_at: f ?? "Unknown",
video_id: d.videoId ?? "Unknown",
video_url: d.video_url ?? "Unknown",
thumb: (d.thumbnails.slice(-1)[0]?.url) ?? "Unknown",
desc: d.description ?? "Unknown"
},
authorInfo: {
channel: {
name: d.author.name ?? "Unknown",
user_tag: d.author.user ?? "Unknown",
id: d.author.id ?? "Unknown",
subs: d.author.subscriber_count ?? "Unknown",
url: d.author.channel_url ?? "Unknown",
url_tag: d.author.user_url ?? "Unknown",
avatar: d.author.thumbnails[2].url ?? "Unknown"
}
},
Infovideo: {
video_url: b.url,
size: Func.bytesToSize(cc)
}
}
};
} catch (error) {
console.error(error);
return {
status: 500,
creator: "David XD",
msg: util.format(error)
};
};
};
};
play = async(q) => {
try {
const a = await yts(q);
const b = a.videos[0];
if (!b) {
return {
status: 404,
creator: "David XD",
msg: "Query " + q + " Not Found :/"
};
};
if (b.seconds >= 7200) {
return {
status: 400,
creator: "David XD",
msg: "Video More Than 2 Hour!"
};
};
const d = (await this.ytmp4(b.url)).data;
const e = d.videoInfo;
const f = d.authorInfo.channel;
const g = d.Infovideo;
const h = (await this.ytmp3(b.url)).data.audioInfo;
return {
status: 200,
creator: "David XD",
data: {
videoInfo: {
title: e.title ?? "Unknown",
views: e.viewCount ?? "Unknown",
duration: e.duration ?? "Unknown",
quality: e.quality ?? "Unknown",
upload_at: e.upload_at ?? "Unknown",
video_id: e.video_id ?? "Unknown",
video_url: e.video_url ?? "Unknown",
thumb: e.thumb ?? "Unknown",
desc: e.desc ?? "Unknown"
},
authorInfo: {
channel: {
name: f.name ?? "Unknown",
user_tag: f.user_tag ?? "Unknown",
id: f.id ?? "Unknown",
subs: f.subs ?? "Unknown",
url: f.url ?? "Unknown",
url_tag: f.url_tag ?? "Unknown",
avatar: f.avatar ?? "Unknown"
}
},
Info: {
video: {
url: g.video_url,
size: g.size
},
audio: {
buffer: h.audio_buffer,
size: h.size
}
}
}
};
} catch (error) {
console.error(error);
return {
status: 500,
creator: "David XD",
msg: util.format(error)
};
};
};
}