@marcosouvereyns/cjdropshipping-product-scraper/src/cjdropshippingProductScraper.js

Get cjdropshipping product details as a json reponse including feedbacks, variants, description, images, etc.,

import puppeteer from "puppeteer"
import cheerio from "cheerio"
import { formatData } from "./formatData.js";

export const defaultLogger = {
	error: console.error,
	warn: console.warn,
	info: console.info,
	log: console.log,
	debug: console.debug
}

async function scrapeProduct({ browser, productUrl, logger, defaultTimeout, loginCookieValue }) {	
	logger.debug(new Date().toISOString(), "Opening New Page")
	const page = await browser.newPage()

	logger.debug(new Date().toISOString(), "Setting Viewport and user agent and default timeout")
	await page.setViewport({ width: 1920, height: 1080 })
	await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36');
	await page.setDefaultTimeout(defaultTimeout)

	logger.debug(new Date().toISOString(), "Setting request interception and listeners")
	await page.setRequestInterception(true)
	
	page.on("request", req => {
		if (["image", "font"].includes(req.resourceType())) {
			req.abort()
		} else {
			req.continue()
		}
	})

	page.on('response', response => {
		const status = response.status()
		const responseUrl = response.url()
		if ((status >= 300) && (status <= 399)) {
			logger.log(`${new Date().toISOString()} Redirect from ${responseUrl} to ${response.headers()['location']}`)
		}
	})

	logger.debug(new Date().toISOString(), "Going to url")
	await page.goto(productUrl, { waitUntil: "domcontentloaded" });

	logger.debug("Fetching productDetailResponse")
	const cjdropshippingData = await page.evaluate(() => window.productDetailResponse);
	const data = cjdropshippingData?.result ?? null;

	await browser.close();

	const json = formatData(data)

	return json;
}

export default async function cjdropshippingProductScraper({ productUrl, logger = defaultLogger, defaultTimeout, loginCookieValue = "" }, tryN = 1) {
	let browser = null
	let json = null

	try {
		logger.log(`Scraping ${productUrl}`)
		logger.debug(new Date().toISOString(), "Launching browser")
		browser = await puppeteer.launch({ defaultViewport: { width: 1920, height: 1080 }, args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080'] })

		const timeBefore = new Date()
		json = await scrapeProduct({ browser, productUrl, logger, defaultTimeout, loginCookieValue })
		const timeAfter = new Date()

		logger.log(`Scraped after ${(timeAfter - timeBefore) / 1000}s`)
		logger.debug("Product title", json.title)
		await browser.close()
	} catch (error) {
		logger.error("cjdropshippingProductScraper Catched error", error)
		if (browser?.close) {
			logger.log("Closing browser after exception")
			await browser.close()
		}

		if (tryN < 3) {
			logger.log(`Retrying to scrape ${productUrl}. Try number ${tryN + 1}`)
			cjdropshippingProductScraper({ productUrl, logger, defaultTimeout }, tryN + 1)
		}
	}
	
	return json
}