nurlresolver/dist/utils/helper.js

url resolver for node

import psl from "psl";
import scrapeIt from "scrape-it";
import got from 'got';
const map = new Map();
export const parseHiddenForm = (html, ix) => parseForms(html).output[ix || 0];
export const parseHiddenFormV2 = (html, ix) => transformScrapedFormToFormData(parseForms(html).output[ix || 0]);
export const getServerPublicIp = async () => {
    const result = await got('https://api.ipify.org?format=json', {
        responseType: 'json',
        resolveBodyOnly: true,
        cache: map //cache it assuming the ip don't change often
    });
    return result.ip;
};
export const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
export const nodeatob = (str) => Buffer.from(str, 'base64').toString();
export const nodebtoa = (str) => Buffer.from(str).toString('base64');
export const parseForms = (html) => {
    const result = scrapeIt.scrapeHTML(html, {
        output: {
            listItem: 'form',
            data: {
                action: {
                    attr: 'action'
                },
                kv: {
                    listItem: 'input',
                    data: {
                        name: {
                            attr: 'name'
                        }, value: {
                            attr: 'value'
                        }
                    }
                }
            }
        }
    });
    return result;
};
export const transformScrapedFormToFormData = (scrapedForm) => {
    if (!scrapedForm?.kv)
        return;
    const form = {};
    for (const { name, value } of scrapedForm.kv) {
        name !== undefined && value !== undefined &&
            name !== null && value !== null && (form[name] = value);
    }
    return form;
};
export const parseAllLinks = (html, context, baseUrl = '') => {
    const result = [];
    const { output } = scrapeIt.scrapeHTML(html, {
        output: {
            listItem: `${context} a`,
            data: {
                title: '',
                link: {
                    attr: 'href'
                }
            }
        }
    });
    output.forEach(x => {
        if (baseUrl && baseUrl.startsWith('http') && !x.link?.startsWith('http')) {
            result.push({
                link: new URL(x.link, baseUrl).href,
                title: x.title
            });
        }
        else if (x.link?.startsWith('http')) {
            result.push({
                link: x.link,
                title: x.title
            });
        }
    });
    return result;
};
export const parseElementAttributes = (html, context, attribute) => {
    const { output } = scrapeIt.scrapeHTML(html, {
        output: {
            listItem: `${context}`,
            data: {
                value: {
                    attr: attribute
                }
            }
        }
    });
    return output.map(x => x.value);
};
export const parseScripts = (html, context = '') => {
    const parsedScripts = scrapeIt.scrapeHTML(html, {
        output: {
            listItem: `${context} script`,
            data: {
                s: {
                    how: 'html'
                }
            }
        }
    });
    return parsedScripts.output.map(x => x.s);
};
export const scrapeLinkHref = (html, selector) => {
    const { link } = scrapeIt.scrapeHTML(html, {
        link: {
            selector: selector,
            attr: 'href'
        }
    });
    return link;
};
export const scrapePageTitle = (html) => {
    const { title } = scrapeIt.scrapeHTML(html, {
        title: 'title'
    });
    return title;
};
export const scrapeInnerText = (html, selector) => {
    const { title } = scrapeIt.scrapeHTML(html, {
        title: selector
    });
    return title;
};
export const getSecondLevelDomain = (someUrl) => {
    const hostname = new URL(someUrl);
    const parsed = psl.parse(hostname.hostname);
    return parsed.sld;
};
export const isValidHttpUrl = (someUrl) => {
    try {
        new URL(someUrl);
    }
    catch (_) {
        return false;
    }
    return true;
};
export const extractFileNameFromUrl = (someUrl) => {
    let fileName = `${new URL(someUrl).pathname.split('/').slice(-1)[0]}`;
    fileName = decodeURIComponent(fileName);
    return fileName;
};
export const parseGoogleFileId = (_url) => {
    const u = new URL(_url);
    if (/(drive|docs|drive\.usercontent)\.google\.com/.test(u.hostname)) {
        if (u.pathname.startsWith('/file/d')) {
            return u.pathname.split('/')[3] || null;
        }
        else if (u.searchParams.get('id')) {
            return u.searchParams.get('id');
        }
    }
    return null;
};
export const simpleCaptcha = (html, listItem) => {
    const result = scrapeIt.scrapeHTML(html, {
        code: {
            listItem: listItem,
            data: {
                codeValue: {},
                codePosition: {
                    attr: 'style',
                    convert: (x) => {
                        const rxResult = /padding-left:(\d*)px/.exec(x);
                        return rxResult && parseInt(rxResult[1]);
                    }
                }
            }
        }
    });
    return result.code
        .sort((a, b) => a.codePosition - b.codePosition)
        .map(x => x.codeValue)
        .join('');
};
//# sourceMappingURL=helper.js.map