nurlresolver
Version:
url resolver for node
102 lines • 4.72 kB
JavaScript
import { BaseUrlResolver } from "../BaseResolver.js";
import * as url from 'url';
export class ExtraMoviesResolver extends BaseUrlResolver {
constructor() {
super({
domains: [/https?:\/\/extramovies/]
});
}
async resolveInner(_urlToResolve) {
const pathname = new URL(_urlToResolve).pathname;
if (pathname === '/')
return [];
//else do the processing here
const links = [];
const responsev1 = await this.gotInstance(_urlToResolve);
const obj1 = this.scrapeHtml(responsev1.body, {
shortLink: {
listItem: 'link',
data: {
rel: {
attr: 'rel'
},
href: {
attr: 'href'
}
}
}
});
const shortLink = obj1.shortLink.find(x => x.rel == 'shortlink')?.href;
if (!shortLink)
throw new Error('Unable to parse shortlink for extramovies resolver');
const urlInstance = url.parse(shortLink, true);
const pageId = urlInstance && urlInstance.query["p"];
if (!pageId)
return links;
const extramoviesBaseUrl = `https://${urlInstance.host}`;
const apiUrl = `${extramoviesBaseUrl}/wp-json/wp/v2/posts/${pageId}`;
const apiResponse = await this.gotInstance(apiUrl).json();
const parsedLinksFromHtml = this.scrapeAllLinks(responsev1.body, '.entry-content');
const renderedContent = apiResponse.content.rendered;
const regex_self_links = /https?:\/\/(extramovies|t.me)/gi;
const knownBadLinks = /https?:\/\/(www\.)imdb\.com/gi;
const obj = this.scrapeAllLinks(renderedContent, '');
const iframeSrc = [...new Set([...this.parseElementAttributes(responsev1.body, 'iframe', 'data-src'),
...this.parseElementAttributes(responsev1.body, 'iframe', 'src'),
...this.parseElementAttributes(renderedContent, 'iframe', 'data-litespeed-src')])]
.filter(this.isValidHttpUrl)
.map(x => {
return { link: x, title: this.getSecondLevelDomain(x) };
});
for (const iterator of [...obj, ...parsedLinksFromHtml, ...iframeSrc]) {
const title = iterator.title;
const link = iterator.link.startsWith('http') ? iterator.link : `${extramoviesBaseUrl}${iterator.link}`;
if (!link || !this.isValidHttpUrl(link)) {
continue;
}
const u = new URL(link);
const regexNegate = /.*(wp-login|torrent|trailer)\.php$/ig;
if (regexNegate.exec(u.pathname))
continue;
if (u.pathname.endsWith('drive.php')) {
const pageText = await this.gotInstance(link);
const regexExtralinks = /https?:\/\/extralinks[^"]*/g;
const matchesLink = regexExtralinks.exec(pageText.body);
if (matchesLink) {
links.push({ title, link: matchesLink[0] });
}
}
else if (u.pathname.endsWith('doodstream.php')) {
const queryData = url.parse(link, true).query;
if (queryData.url) {
const linktoadd = `https://dood.watch/d/${queryData.url}`;
links.push({ title, link: linktoadd });
}
}
else if (u.pathname.endsWith('.php')) {
const queryData = url.parse(link, true).query;
if (queryData.link) {
const linktoadd = Buffer.from(queryData.link, 'base64').toString();
links.push({ title, link: linktoadd });
}
else if (queryData.id) {
const responseInner2 = await this.gotInstance(link);
const result = this.scrapeAllLinks(responseInner2.body, '', responseInner2.url);
result.forEach(instanceItem => instanceItem.title = title);
links.push(...result.filter(l => l.link && !l.link.match(regex_self_links)));
}
}
else if (u.host.startsWith('extramovies')) {
//just skip if the link is linking back to some more extramovies link
}
else if (u.pathname.startsWith('/category')) {
//just skip if the link is linking back to some /category/
}
else {
links.push({ title, link });
}
}
return links.filter(x => !x.link.match(knownBadLinks));
}
}
//# sourceMappingURL=extraMoviesResolver.js.map