universal-pdp-scrapper
Version:
A universal pdp scrapper using cheerio & ChatGPT
1,162 lines (1,136 loc) • 54.4 kB
JavaScript
import { load } from 'cheerio';
import { config } from 'dotenv';
import { encodingForModel } from 'js-tiktoken';
import { OpenAI } from 'openai';
import axios from 'axios';
import { customsearch } from '@googleapis/customsearch';
import { getJson } from 'serpapi';
var name = "universal-pdp-scrapper";
var version = "1.2.4";
var description = "A universal pdp scrapper using cheerio & ChatGPT";
var repository = {
type: "git",
url: "git+https://github.com/subho57/universal-pdp-scrapper.git"
};
var homepage = "https://subho57.github.io/universal-pdp-scrapper/docs/";
var sideEffects = false;
var publishConfig = {
access: "public"
};
var bugs = {
url: "https://github.com/subho57/universal-pdp-scrapper/issues"
};
var keywords = [
"universal-scrapper",
"product-detail-page-scrapper",
"typescript",
"scrapper",
"cheerio",
"OpenAI-API",
"ChatGPT",
"chat-gpt-scrapper",
"ikea-scrapper",
"3d-model-scrapper",
"amazon-scrapper",
"subho57",
"better-than-scrapegraph",
"ai-scrapper",
"scrapegraph-alternative",
"build.com-scrapper"
];
var source = "src/index.ts";
var main = "dist/index.js";
var unkpg = "dist/index.umd.js";
var module = "dist/index.mjs";
var amdName = "UniversalPDPScrapper";
var exports = {
types: "./dist/index.d.ts",
require: "./dist/index.js",
"default": "./dist/index.modern.mjs"
};
var types = "dist/index.d.ts";
var typings = "dist/index.d.ts";
var engines = {
node: ">=18.0.0",
bun: ">=1.1.0"
};
var scripts = {
"build-types": "tsc --noEmit --pretty",
build: "rimraf docs dist && microbundle --target node --visualize && microbundle --target web --no-compress --format umd --external none && typedoc src/index.ts && cp stats.html docs && mv dist/src/* ./dist/ && rimraf dist/src",
start: "microbundle watch",
prepare: "husky install && npm i -g bun",
postinstall: "husky install",
prepack: "pinst --disable",
postpack: "pinst --enable",
"clean-install": "rm -rf node_modules bun.lockb && bun install",
test: "env-cmd jest > test-report.log 2>&1 && echo 'Test Report: ./test-report.log'",
"test:build": "run-s build",
lint: "eslint --ext .js,.ts .",
"lint-fix": "eslint --fix --ext .js,.ts ."
};
var author = "subho57";
var license = "MIT";
var packageManager = "bun@1.1.43";
var dependencies = {
"@googleapis/customsearch": "^3.2.0",
axios: "^1.7.9",
cheerio: "^1.0.0",
dotenv: "^16.4.7",
"js-tiktoken": "^1.0.16",
openai: "^4.78.1",
serpapi: "^2.1.0"
};
var devDependencies = {
"@commitlint/cli": "^19.6.1",
"@commitlint/config-conventional": "^19.6.0",
"@types/jest": "^29.5.14",
"@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.62.0",
"env-cmd": "^10.1.0",
eslint: "^8.57.1",
"eslint-config-airbnb-base": "^15.0.0",
"eslint-config-airbnb-typescript": "^18.0.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.31.0",
"eslint-plugin-jest": "^28.11.0",
"eslint-plugin-jest-formatting": "^3.1.0",
"eslint-plugin-prettier": "^5.2.2",
"eslint-plugin-simple-import-sort": "^12.1.1",
"eslint-plugin-unused-imports": "^4.1.4",
"gh-pages": "^6.3.0",
husky: "^9.1.7",
jest: "^29.7.0",
"lint-staged": "^15.3.0",
microbundle: "^0.15.1",
pinst: "^3.0.0",
prettier: "^3.4.2",
rimraf: "^6.0.1",
"ts-jest": "^29.2.5",
typedoc: "^0.27.6",
typescript: "^5.7.3"
};
var files = [
"dist"
];
var pkg = {
name: name,
version: version,
description: description,
repository: repository,
homepage: homepage,
sideEffects: sideEffects,
publishConfig: publishConfig,
bugs: bugs,
keywords: keywords,
source: source,
main: main,
"umd:main": "dist/index.umd.js",
unkpg: unkpg,
module: module,
amdName: amdName,
exports: exports,
types: types,
typings: typings,
engines: engines,
scripts: scripts,
author: author,
license: license,
packageManager: packageManager,
dependencies: dependencies,
devDependencies: devDependencies,
files: files
};
var _process$env$OPEN_AI_;
config();
var CONFIG = {
APP: {
NAME: pkg.name,
VERSION: pkg.version,
DESCRIPTION: pkg.description,
AUTHOR: pkg.author,
STAGE: process.env.NODE_ENV === 'development' ? 'dev' : 'prod',
ENV: process.env.NODE_ENV || 'development'
},
SERP: {
API_KEY: process.env.SERP_API_KEY || ''
},
OPEN_AI: {
API_KEY: (_process$env$OPEN_AI_ = process.env.OPEN_AI_SECRET_KEY) != null ? _process$env$OPEN_AI_ : '',
ORG_ID: process.env.OPEN_AI_ORG_ID || undefined,
MODEL: process.env.OPEN_AI_MODEL || 'gpt-4o-mini'
},
GOOGLE: {
API_KEY: process.env.GOOGLE_API_KEY || '',
CSE_ID: process.env.GOOGLE_CSE_ID || ''
}
};
class Logger {
constructor(name) {
this.name = void 0;
this.name = name;
}
info(message, ...args) {
console.log(`[${this.name}] ${message}:`, ...args);
}
error(message, ...args) {
console.error(`[${this.name}] ${message}:`, ...args);
}
warn(message, ...args) {
console.warn(`[${this.name}] ${message}:`, ...args);
}
debug(message, ...args) {
console.debug(`[${this.name}] ${message}:`, ...args);
}
trace(message, ...args) {
console.trace(`[${this.name}] ${message}:`, ...args);
}
}
const capitalize = s => {
if (!s) {
return 'Unnamed room';
}
s = s.replace(/\//g, ' ');
s = s.replace(/:/g, ' ');
s = s.replace(/,/g, ' ');
s = s.replace(/-/g, ' ');
s = s.replace(/_/g, ' ');
s = s.replaceAll(' ', ' ');
s = s.replace(/(^\w{1})|(\s+\w{1})/g, letter => letter.toUpperCase());
s = s.replace(/[\d.]+$/, '');
return s.trim();
};
function removeNullsAndUndefined(obj) {
for (const propName in obj) {
if (obj[propName] === null || obj[propName] === undefined || obj[propName] === '') {
delete obj[propName];
}
}
}
function parseJSONFromMarkdownString(mdString, options = {
throwOnError: true
}) {
try {
if (!mdString || typeof mdString !== 'string') {
throw new Error('Input must be a non-empty string');
}
const normalizedString = mdString.replace(/\r\n/g, '\n');
let jsonString = normalizedString;
if (normalizedString.includes('```json')) {
const jsonBlockRegex = /```json\s*([\s\S]*?)\s*```/;
const matches = normalizedString.match(jsonBlockRegex);
if (!matches || !matches[1]) {
throw new Error('Invalid markdown JSON format');
}
[, jsonString] = matches;
}
const trimmedString = jsonString.trim();
const parsed = JSON.parse(trimmedString);
if (parsed === undefined) {
throw new Error('Parsing resulted in undefined value');
}
return parsed;
} catch (error) {
if (options.throwOnError) {
throw new Error(`Failed to parse JSON: ${error.message} => ${mdString}`);
}
return options.defaultValue;
}
}
async function getHtml(url) {
const parsedUrl = new URL(url);
let {
data: html
} = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
Accept: 'application/json, text/plain, */*',
'Accept-Encoding': 'gzip, deflate, br',
'Content-Type': 'application/json',
'X-Requested-With': 'XMLHttpRequest',
Origin: parsedUrl.origin,
Host: parsedUrl.host,
Connection: 'keep-alive'
}
}).catch(err => {
console.debug('getHtml.axios.get', err.message, err.response);
return {
data: '<html><head><title></title></head></html>'
};
});
if (typeof html !== 'string') {
html = JSON.stringify(html);
}
return html;
}
const logger$3 = new Logger('OpenAIService');
const getClient = config => new OpenAI(config);
const getCompletion = async (prompt, client, model = 'gpt-4o-mini') => {
try {
var _result$choices$;
const result = await client.chat.completions.create({
model,
messages: [{
role: 'system',
content: prompt
}],
n: 1,
temperature: 0
});
return (_result$choices$ = result.choices[0]) == null || (_result$choices$ = _result$choices$.message) == null ? void 0 : _result$choices$.content;
} catch (err) {
logger$3.error('getCompletion', err.message);
return undefined;
}
};
const trimToTokenLimit = (text, enc, maxTokens = 128000, separator = '-------') => {
if (!text || !enc) {
return text;
}
const tokens = enc.encode(text);
if (tokens.length <= maxTokens) {
return text;
}
if (text.includes(separator)) {
const parts = text.split(separator);
if (parts.length !== 3) {
return enc.decode(tokens.slice(0, maxTokens));
}
const [beforeHtml, html, afterHtml] = parts;
const beforeTokens = enc.encode(beforeHtml).length;
const afterTokens = enc.encode(afterHtml).length;
const intermediatoryTokens = enc.encode(`${separator}...[content trimmed]...${separator}`).length;
const availableTokens = maxTokens - beforeTokens - afterTokens - intermediatoryTokens;
if (availableTokens < 0) {
return enc.decode(tokens.slice(0, maxTokens));
}
const htmlTokens = enc.encode(html);
const halfAvailable = Math.floor(availableTokens / 2);
const startHtml = enc.decode(htmlTokens.slice(0, halfAvailable));
const endHtml = enc.decode(htmlTokens.slice(-halfAvailable));
return `${beforeHtml}${separator}${startHtml}...[content trimmed]...${endHtml}${separator}${afterHtml}`;
}
return enc.decode(tokens.slice(0, maxTokens));
};
const scrapeUsingAI = async (client, productUrl, htmlContent, userId, metadata, model = 'gpt-4o-mini', trimToMaxToken = 112000, timeoutSeconds = 29) => {
try {
var _result$choices$3;
const enc = encodingForModel(model);
const systemContext = `You are tasked with a job to accept from a given product URL and optionally its HTML content from the user and return the extracted information in a structured JSON format. You should not prompt the user to provide any additional details; rather, work with the details that are already provided to you.
You should be able to extract the following information:
- **Product Name**
- **Description**: A string describing the product in one line, such that when this string is searched in Google, it shows similar products.
- **Tags**: A list of comma-separated words relating to the product.
- **Product URL**
- **Type**: One of the following: bathtub, bed, bench, bookshelf, cabinet, chair, coffee_table, console_table, day_bed, desk, dressing_table, faucet, floor_lamp, futon, light, loveseat, mattress, mirror, nightstand, painting, plant_stand, rug, side_table, sink, sleeper_sofa, sofa, sofa_bed, stool, storage_unit, study_table, table_lamp, toy, tv_stand, vanity, wall_lamp, wall_mirror, wall_shelf, water_closet.
- **Price**: Converted to US dollars in format XXX.XX.
- **Exact Height**: In inches and up to one decimal place, in XX.X format with unit converted to inches.
- **Exact Width**: In inches and up to one decimal place, in XX.X format with unit converted to inches.
- **Exact Depth**: In inches and up to one decimal place, in XX.X format with unit converted to inches.
- **Images**: A list of image URLs.
- **SKU**
- **Source**: Brand name.
The information should be structured properly in the following JSON schema:
{
"product_name": string,
"product_url": string,
"type": string,
"price": number,
"height": number,
"width": number,
"depth": number,
"tags": string,
"images": string[],
"sku": string,
"source": string,
"description": string
}
# Output Format
Strictly return the extracted information in plain JSON format without any plain text and without any markdown formatting.
# Examples
Example JSON object for the product URL: [https://www.ikea.com/us/en/p/haegernaes-table-and-4-chairs-antique-stain-pine-70575947/]
{
"product_name": "HÄGERNÄS",
"product_url": "https://www.ikea.com/us/en/p/haegernaes-table-and-4-chairs-antique-stain-pine-70575947/",
"type": "table",
"price": 199.99,
"height": 29.50,
"width": 59.10,
"depth": 35.40,
"tags": "table, chairs, dining, antique, pine, furniture",
"images": [
"https://www.ikea.com/us/en/images/products/haegernaes-table-and-4-chairs-antique-stain-pine__1350925_pe951817_s5.jpg"
],
"sku": "70575947",
"source": "ikea",
"description": "This sturdy dining set with a table and four chairs is perfect for your breakfast nook or smaller dining area. Solid pine is a natural material that ages beautifully and acquires its own unique character over time. Each table and chair has its own unique character due to the distinctive grain pattern. For a softer seat or to add a personal touch to the room, complete with a chair pad in the style and color of your choice."
}`;
let userPrompt = `Here's the product URL: ${productUrl}${htmlContent ? `\nand the HTML content:
-------
${htmlContent}
-------
` : ''}`.substring(0, 1048575);
const reservedTokens = enc.encode(systemContext).length;
const availableTokens = trimToMaxToken - reservedTokens;
userPrompt = trimToTokenLimit(userPrompt, enc, availableTokens);
const apiCallPromise = client.chat.completions.create({
model,
messages: [{
role: 'system',
content: systemContext
}, {
role: 'user',
content: userPrompt
}],
n: 1,
temperature: 0,
response_format: {
type: 'json_object'
},
stream: false,
user: userId,
metadata
});
if (timeoutSeconds) {
var _result$choices$2;
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => {
reject(new Error(`Operation timed out after ${timeoutSeconds} seconds`));
}, timeoutSeconds * 1000);
});
const result = await Promise.race([apiCallPromise, timeoutPromise]);
return parseJSONFromMarkdownString(((_result$choices$2 = result.choices[0]) == null || (_result$choices$2 = _result$choices$2.message) == null ? void 0 : _result$choices$2.content) || '');
}
const result = await apiCallPromise;
return parseJSONFromMarkdownString(((_result$choices$3 = result.choices[0]) == null || (_result$choices$3 = _result$choices$3.message) == null ? void 0 : _result$choices$3.content) || '');
} catch (error) {
logger$3.error('scrapeUsingAI', error instanceof Error ? error.message : 'Unknown error');
return {};
}
};
class Article {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
var _$$html, _dimension$split$, _dimension$split$2, _dimension$split$3, _$$text;
if (!html) {
if (!this.url.includes('article')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const dimension = (_$$html = $('#details > div > div.content.desktop.flex-grid > div:nth-child(3) > div > div').html()) == null || (_$$html = _$$html.match(/[\d.]+"H x [\d.]+"W x [\d.]+"D/gm)) == null ? void 0 : _$$html[0];
const res = {
product_name: $('#app > div > div.app-container-wrapper > div > div > div.product-nav > div.product-nav-desktop.container > div > div.primary-info > div > h2').text().replace(/[\r\n]/gm, '').trim(),
images: $('#app > div > div.app-container-wrapper > div > div > div.main-layout > div.product-buy-section-container.container.product-page-buy-section > div > div.product-buy-images > div.product-thumbnails-container > div > a.active.product-thumbnail > div > picture > img').map((_i, el) => {
var _$$attr, _$$attr2;
return (_$$attr = $(el).attr('src')) == null ? void 0 : _$$attr.substring(0, (_$$attr2 = $(el).attr('src')) == null ? void 0 : _$$attr2.indexOf('?'));
}).get(),
depth: dimension == null || (_dimension$split$ = dimension.split('x')[2]) == null || (_dimension$split$ = _dimension$split$.trim()) == null ? void 0 : _dimension$split$.replace(/\D+/g, ''),
width: dimension == null || (_dimension$split$2 = dimension.split('x')[1]) == null || (_dimension$split$2 = _dimension$split$2.trim()) == null ? void 0 : _dimension$split$2.replace(/\D+/g, ''),
height: dimension == null || (_dimension$split$3 = dimension.split('x')[0]) == null || (_dimension$split$3 = _dimension$split$3.trim()) == null ? void 0 : _dimension$split$3.replace(/\D+/g, ''),
price: (_$$text = $('#app > div > div.app-container-wrapper > div > div > div.product-nav > div.product-nav-desktop.container > div > div.product-navbar-desktop > div.sale-price > div > span').text()) == null || (_$$text = _$$text.replace(/[\r\n]/gm, '')) == null || (_$$text = _$$text.replace(/\D+/g, '')) == null ? void 0 : _$$text.trim(),
source: 'article',
product_url: this.url
};
return res;
}
}
class Builddotcom {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
if (!html) {
if (!this.url.includes('build.com')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const productJson = JSON.parse($('#main-content > div.center-ns.mw9-ns.b--theme-grey.bg-theme-white.ba-ns.pa2.pa3-ns.mt3-ns.mb3 > script').text());
const thumbnail = productJson.image.replace('t_base,c_lpad,f_auto,dpr_auto,w_1200,h_1200', 't_base');
const productName = $('#pdp-buysection > div.cf.flex.db-ns.flex-column > div.fl.w-50-ns.order-0 > section > h1 > span.fw2.di-ns').text().replace(/[\r\n]/gm, '').trim();
const res = {
product_name: productJson.name,
images: thumbnail ? [thumbnail] : [],
price: productJson.offers.price.toString(),
source: 'build.com',
sku: productJson.offers.sku,
product_url: productJson.offers.url,
artist: productJson.brand.name,
description: productName,
'supporting-surface': productJson.description.toLowerCase().includes('wall') ? 'wall' : 'floor'
};
return res;
}
}
class Etsy {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
var _$$filter$text, _$$filter$text2, _$$filter$text3, _$$text;
if (!html) {
if (!this.url.includes('etsy')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const res = {
product_name: $('#listing-page-cart > div.wt-mb-xs-2 > h1').text().replace(/[\r\n]/gm, '').trim(),
images: $('#listing-right-column > div > div.body-wrap.wt-body-max-width.wt-display-flex-md.wt-flex-direction-column-xs > div.image-col.wt-order-xs-1.wt-mb-lg-6 > div > div > div.image-wrapper.wt-position-relative.carousel-container-responsive > div > div.image-carousel-container.wt-position-relative.wt-flex-xs-6.wt-order-xs-2.show-scrollable-thumbnails > ul > li > img').map((_i, el) => $(el).attr('data-src-zoom-image')).get(),
height: (_$$filter$text = $('#product-details-content-toggle > div > ul > div > div:nth-last-child(1) > li > div').filter((_i, el) => $(el).text().includes('Height')).text()) == null || (_$$filter$text = _$$filter$text.split(':')[1]) == null || (_$$filter$text = _$$filter$text.trim()) == null ? void 0 : _$$filter$text.replace(/\D+/g, ''),
width: (_$$filter$text2 = $('#product-details-content-toggle > div > ul > div > div:nth-last-child(1) > li > div').filter((_i, el) => $(el).text().includes('Width')).text()) == null || (_$$filter$text2 = _$$filter$text2.split(':')[1]) == null || (_$$filter$text2 = _$$filter$text2.trim()) == null ? void 0 : _$$filter$text2.replace(/\D+/g, ''),
depth: (_$$filter$text3 = $('#product-details-content-toggle > div > ul > div > div:nth-last-child(1) > li > div').filter((_i, el) => $(el).text().includes('Depth')).text()) == null || (_$$filter$text3 = _$$filter$text3.split(':')[1]) == null || (_$$filter$text3 = _$$filter$text3.trim()) == null ? void 0 : _$$filter$text3.replace(/\D+/g, ''),
material: (_$$text = $('#legacy-materials-product-details').text()) == null || (_$$text = _$$text.split(':').pop()) == null ? void 0 : _$$text.trim(),
price: $('#listing-page-cart > div.wt-mb-xs-6.wt-mb-lg-0 > div:nth-child(1) > div.wt-mb-xs-3 > div.wt-mb-xs-3 > div.wt-display-flex-xs.wt-align-items-center.wt-justify-content-space-between > div.wt-display-flex-xs.wt-align-items-center > p > span:nth-child(2)').text().replace(/[\r\n]/gm, '').trim(),
sku: this.url.split('/')[this.url.split('/').length - 2],
artist: $('#desktop_shop_owners_parent > div > div > div.wt-display-flex-xs.wt-align-items-center.wt-mb-xs-2 > div:nth-child(2) > p.wt-text-body-03.wt-line-height-tight.wt-mb-xs-1').text().trim(),
source: 'etsy',
product_url: this.url
};
return res;
}
}
class Fineartamerica {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
var _$$attr$trim, _$$attr, _$$attr2;
if (!html) {
if (!this.url.includes('fineartamerica')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const res = {
product_name: $('#h1title').text().trim(),
images: [(_$$attr$trim = (_$$attr = $('#productPreviewImage').attr('src')) == null ? void 0 : _$$attr.trim()) != null ? _$$attr$trim : (_$$attr2 = $('#mainimage').attr('src')) == null ? void 0 : _$$attr2.trim()].filter(x => !!x),
material: $('#priceDetailDiv > div:nth-child(2) > p:nth-child(2)').text().trim(),
price: `${$('#productCurrency').text().trim()} ${$('#productPrice').text().trim()}`,
artist: $('#artistName > a').text().trim(),
source: 'fineartamerica',
product_url: this.url
};
return res;
}
}
const logger$2 = new Logger('googleScrapper');
class Google {
constructor(config) {
this.serpApiKey = void 0;
this.googleApiKey = '';
this.googleCseId = '';
this.url = void 0;
this.url = config.url;
if ('serpApiKey' in config) {
this.serpApiKey = config.serpApiKey;
} else {
this.googleApiKey = config.googleApiKey;
this.googleCseId = config.googleCseId;
}
}
async getImages(prompt, useSerpApi) {
var _results$images_resul, _results$data$items$m, _results$data$items;
const results = useSerpApi ? await getJson('google', {
api_key: this.serpApiKey,
q: prompt,
gl: 'us',
hl: 'en',
tbm: 'isch',
filter: '1',
google_domain: 'google.com',
location: 'California, United States'
}).catch(err => {
logger$2.error('getImages.usingSerpApi', err);
return undefined;
}) : await customsearch('v1').cse.list({
auth: this.googleApiKey,
q: prompt,
cr: 'countryUS',
cx: this.googleCseId,
gl: 'us',
hl: 'en',
filter: '1',
num: 5,
safe: 'active',
searchType: 'image'
}, {
http2: true
}).catch(err => {
logger$2.error('getImages.usingGoogleApi', err);
return undefined;
});
return useSerpApi ? ((_results$images_resul = results == null ? void 0 : results.images_results.slice(0, 3)) != null ? _results$images_resul : []).map(image => image.original) : (_results$data$items$m = results == null || (_results$data$items = results.data.items) == null ? void 0 : _results$data$items.map(item => item.link)) != null ? _results$data$items$m : [];
}
async extract(html) {
try {
var _domainParts$at;
if (!html) {
html = await getHtml(this.url);
}
const parsedURL = new URL(this.url);
const domain = parsedURL.hostname.replace('www.', '');
const domainParts = domain.split('.');
const pathName = parsedURL.pathname.replaceAll('/', ' ').trim();
const $ = load(html);
const title = `${capitalize(`${pathName} ${$('head > title').text().trim()}`)}`.toLowerCase().replace(/\b(?![\w\d]+\b)[^\s]+\b/g, '').replace((_domainParts$at = domainParts.at(-2)) != null ? _domainParts$at : '', '').replace(/\s{2,}/g, ' ').split(' ').filter(word => !(word.includes('0') || word.includes('1') || word.includes('2') || word.includes('3') || word.includes('4') || word.includes('5') || word.includes('6') || word.includes('7') || word.includes('8') || word.includes('9') || word.includes('&') || word.includes('|') || word.includes('='))).join(' ').trim();
const processedTitle = Array.from(new Set(`${title} from ${domain}`.toLowerCase().split(' '))).join(' ');
const images = await this.getImages(processedTitle, !!this.serpApiKey);
if (!images.length && this.serpApiKey && this.googleApiKey && this.googleCseId) {
logger$2.warn('No images found, trying without serp api');
images.push(...(await this.getImages(processedTitle, false)));
}
const res = {
images,
product_name: capitalize(title),
source: domainParts.at(-2),
product_url: this.url
};
return res;
} catch (error) {
var _error$message;
logger$2.error('extract', (_error$message = error == null ? void 0 : error.message) != null ? _error$message : error);
throw error;
}
}
}
var BrandType;
(function (BrandType) {
BrandType["Brand"] = "Brand";
BrandType["Person"] = "Person";
})(BrandType || (BrandType = {}));
var ReviewType;
(function (ReviewType) {
ReviewType["Review"] = "Review";
})(ReviewType || (ReviewType = {}));
var ReviewRatingType;
(function (ReviewRatingType) {
ReviewRatingType["Rating"] = "Rating";
})(ReviewRatingType || (ReviewRatingType = {}));
class Homedepot {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
if (!html) {
if (!this.url.includes('homedepot')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const stringifiedLdJson = $('#thd-helmet__script--productStructureData').html();
const ldJson = JSON.parse(stringifiedLdJson != null ? stringifiedLdJson : '');
return {
product_name: ldJson.name,
images: ldJson.image.map(img => img.replace('_100', '_1024')),
height: ldJson.height,
width: ldJson.width,
depth: ldJson.depth,
material: ldJson.color,
price: ldJson.offers.price.toString(),
sku: ldJson.sku,
artist: ldJson.brand.name,
product_url: this.url,
source: 'homedepot',
description: ldJson.description,
tags: 'furniture'
};
}
}
const logger$1 = new Logger('IkeaScrapper');
class Ikea {
constructor(url) {
this.url = void 0;
this.sku = void 0;
this.url = url;
const {
pathname
} = new URL(this.url);
this.sku = pathname.slice(1).replace(/\/$/, '').split('-').pop().replace(/[^\d]/g, '');
}
async getProductDetails() {
try {
return await axios.get(`${Ikea.BASE_URL}/us/en/products/${this.sku.slice(-3)}/${this.sku}.json`).then(response => response.data);
} catch (err) {
var _err$response, _err$response2;
logger$1.error('getProductDetails', err == null ? void 0 : err.message, (_err$response = err.response) == null ? void 0 : _err$response.status, (_err$response2 = err.response) == null ? void 0 : _err$response2.data);
return null;
}
}
async getGLBs() {
try {
return await axios.get(`${Ikea.BASE_URL}/global/assets/rotera/resources/${this.sku}.json`).then(response => response.data);
} catch (err) {
var _err$response3, _err$response4;
logger$1.debug('getGLBs', err == null ? void 0 : err.message, (_err$response3 = err.response) == null ? void 0 : _err$response3.status, (_err$response4 = err.response) == null ? void 0 : _err$response4.data);
return null;
}
}
async extract(html) {
if (!html) {
if (!this.url.includes('ikea')) {
throw new Error('Invalid URL');
}
const response = await axios.get(this.url);
html = response.data;
}
const [productDetails, glbs] = await Promise.allSettled([this.getProductDetails(), this.getGLBs()]).then(resolvedPromises => resolvedPromises.map(p => {
if (p.status === 'fulfilled') {
return p.value;
}
return null;
}));
let res = {};
if (!productDetails || Object.keys(productDetails).length === 0) {
var _$$filter$text, _$$filter$text2, _$$filter$text3, _glbs$models$map;
const $ = load(html);
res = {
product_name: $('#pip-buy-module-content > div.pip-temp-price-module.pip-temp-price-module--informational.pip-temp-price-module--small.pip-temp-price-module--regular-price-package > div.pip-temp-price-module__information > div > h1 > span.pip-header-section__title--big.notranslate').text().replace(/[\r\n]/gm, '').trim(),
images: $('.pip-image').map((_i, el) => $(el).attr('src')).get(),
depth: (_$$filter$text = $('#range-modal-mount-node > div > div:nth-child(3) > div > div.pip-sheets__content-wrapper > div > div > div > div.pip-product-dimensions__dimensions-container > p').filter((_i, el) => {
var _$$html;
return !!((_$$html = $(el).html()) != null && _$$html.includes('Length'));
}).text()) == null || (_$$filter$text = _$$filter$text.split(' ')[0]) == null || (_$$filter$text = _$$filter$text.trim()) == null ? void 0 : _$$filter$text.replace(/\D+/g, ''),
width: (_$$filter$text2 = $('#range-modal-mount-node > div > div:nth-child(3) > div > div.pip-sheets__content-wrapper > div > div > div > div.pip-product-dimensions__dimensions-container > p').filter((_i, el) => {
var _$$html2;
return !!((_$$html2 = $(el).html()) != null && _$$html2.includes('Width'));
}).text()) == null || (_$$filter$text2 = _$$filter$text2.split(' ')[0]) == null || (_$$filter$text2 = _$$filter$text2.trim()) == null ? void 0 : _$$filter$text2.replace(/\D+/g, ''),
height: (_$$filter$text3 = $('#range-modal-mount-node > div > div:nth-child(3) > div > div.pip-sheets__content-wrapper > div > div > div > div.pip-product-dimensions__dimensions-container > p').filter((_i, el) => {
var _$$html3;
return !!((_$$html3 = $(el).html()) != null && _$$html3.includes('height'));
}).text()) == null || (_$$filter$text3 = _$$filter$text3.split(' ')[0]) == null || (_$$filter$text3 = _$$filter$text3.trim()) == null ? void 0 : _$$filter$text3.replace(/\D+/g, ''),
price: $('#pip-buy-module-content > div.pip-temp-price-module.pip-temp-price-module--informational.pip-temp-price-module--small.pip-temp-price-module--regular-price-package > div.pip-temp-price-module__price > div > span > span:nth-child(1) > span.pip-temp-price__integer').text().replace(/[\r\n]/gm, '').trim(),
sku: this.sku,
source: 'ikea',
product_url: this.url,
glbs: (_glbs$models$map = glbs == null ? void 0 : glbs.models.map(model => model.url)) != null ? _glbs$models$map : Array.from(new Set(html.match(/https:\/\/web-api\.ikea\.com\/dimma\/assets\/.[^"]*\.glb/g))).sort()
};
} else {
var _productDetails$mainI, _productDetails$price, _glbs$models$map2;
res = {
product_name: productDetails == null ? void 0 : productDetails.name,
images: productDetails != null && (_productDetails$mainI = productDetails.mainImage) != null && _productDetails$mainI.url ? [productDetails.mainImage.url] : [],
price: productDetails == null || (_productDetails$price = productDetails.priceNumeral) == null ? void 0 : _productDetails$price.toString(),
sku: this.sku,
product_url: this.url,
source: 'ikea',
glbs: (_glbs$models$map2 = glbs == null ? void 0 : glbs.models.map(model => model.url)) != null ? _glbs$models$map2 : []
};
}
return res;
}
}
Ikea.BASE_URL = 'https://www.ikea.com';
class Rugsdotcom {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
var _$$text, _$$text2;
if (!html) {
if (!this.url.includes('rugs.com')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const thumbnail = $('#FullScreenImageGallery > div:nth-child(3) > div > img').attr('src');
const res = {
product_name: $('#react-root > div.row.product-display--content-container > div.col-12.col-md-6.pl-md-0.pr-xl-0.product-display--description > div:nth-child(2) > div > div:nth-child(1) > div > h1').text().replace(/[\r\n]/gm, '').trim(),
images: thumbnail ? [thumbnail.substring(0, thumbnail.indexOf('?'))] : [],
price: (_$$text = $('#add_to_cart_form > div:nth-child(1) > div > div > div.col-12.px-md-0.d-flex.align-items-center > span.h2-bold-no-margin.price.mr-2').text()) == null || (_$$text = _$$text.replace(/[\r\n]/gm, '')) == null || (_$$text = _$$text.replace(/\D+/g, '')) == null ? void 0 : _$$text.trim(),
source: 'rugs.com',
sku: (_$$text2 = $('#react-root > div.row.d-none.d-md-flex.py-3.product-display--breadcrumbs.width-fixed > div > strong').text()) == null ? void 0 : _$$text2.trim(),
product_url: this.url
};
return res;
}
}
class Zgallerie {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
var _result$1$substring, _result$, _info$props$pageProps, _info$props$pageProps2, _info$props$pageProps3, _info$props$pageProps4, _info$props$pageProps5;
if (!html) {
if (!this.url.includes('zgallerie')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const re = /<script id="__NEXT_DATA__" type="application\/json">(.+)<\/script>/;
const result = html.match(re);
const info = JSON.parse((_result$1$substring = result == null || (_result$ = result[1]) == null ? void 0 : _result$.substring(0, result[1].indexOf('</script>'))) != null ? _result$1$substring : '');
const res = {
product_name: info.props.pageProps.product.name.split('~')[0],
images: info.props.pageProps.product.images.map(image => image.url_zoom),
height: (_info$props$pageProps = info.props.pageProps.product.height) == null ? void 0 : _info$props$pageProps.toString(),
width: (_info$props$pageProps2 = info.props.pageProps.product.width) == null ? void 0 : _info$props$pageProps2.toString(),
depth: (_info$props$pageProps3 = info.props.pageProps.product.depth) == null ? void 0 : _info$props$pageProps3.toString(),
price: (_info$props$pageProps4 = info.props.pageProps.product.calculated_price) == null ? void 0 : _info$props$pageProps4.toString(),
sku: (_info$props$pageProps5 = info.props.pageProps.product.sku) == null ? void 0 : _info$props$pageProps5.toString(),
artist: info.props.pageProps.product.additional_info.artist_name ? info.props.pageProps.product.additional_info.artist_name[0] : '',
source: 'zgallerie',
product_url: this.url
};
return res;
}
}
var Types;
(function (Types) {
Types["BATHTUB"] = "bathtub";
Types["BED"] = "bed";
Types["BENCH"] = "bench";
Types["BOOKSHELF"] = "bookshelf";
Types["CABINET"] = "cabinet";
Types["CHAIR"] = "chair";
Types["COFFEE_TABLE"] = "coffee_table";
Types["CONSOLE_TABLE"] = "console_table";
Types["DAY_BED"] = "day_bed";
Types["DESK"] = "desk";
Types["DRESSING_TABLE"] = "dressing_table";
Types["FAUCET"] = "faucet";
Types["FLOOR_LAMP"] = "floor_lamp";
Types["FUTON"] = "futon";
Types["LIGHT"] = "light";
Types["LOVESEAT"] = "loveseat";
Types["MATTRESS"] = "mattress";
Types["MIRROR"] = "mirror";
Types["NIGHTSTAND"] = "nightstand";
Types["PAINTING"] = "painting";
Types["PLANT_STAND"] = "plant_stand";
Types["RUG"] = "rug";
Types["SIDE_TABLE"] = "side_table";
Types["SINK"] = "sink";
Types["SLEEPER_SOFA"] = "sleeper_sofa";
Types["SOFA"] = "sofa";
Types["SOFA_BED"] = "sofa_bed";
Types["STOOL"] = "stool";
Types["STORAGE_UNIT"] = "storage_unit";
Types["STUDY_TABLE"] = "study_table";
Types["TABLE_LAMP"] = "table_lamp";
Types["TOY"] = "toy";
Types["TV_STAND"] = "tv_stand";
Types["VANITY"] = "vanity";
Types["WALL_LAMP"] = "wall_lamp";
Types["WALL_MIRROR"] = "wall_mirror";
Types["WALL_SHELF"] = "wall_shelf";
Types["WATER_CLOSET"] = "water_closet";
})(Types || (Types = {}));
const logger = new Logger('ScrapperClient');
const determineType = searchString => {
try {
const allTags = capitalize(searchString).toLocaleLowerCase();
const allModelTypes = Object.values(Types);
const determinedType = allModelTypes.find(type => {
const typeTags = type.toLocaleLowerCase().split('_');
return typeTags.some(tag => allTags.includes(tag));
});
return determinedType;
} catch (err) {
logger.error('determineType', err.message);
return undefined;
}
};
const isAnImage = html => {
return !html.includes('<html');
};
const getRawData = async (url, html) => {
try {
if (isAnImage(html)) {
var _url$split$pop;
return {
images: [url],
product_url: url,
product_name: (_url$split$pop = url.split('/').pop()) == null ? void 0 : _url$split$pop.split('.').shift(),
source: 'user-sourced'
};
}
const googleClient = new Google({
url,
serpApiKey: CONFIG.SERP.API_KEY,
googleApiKey: CONFIG.GOOGLE.API_KEY,
googleCseId: CONFIG.GOOGLE.CSE_ID
});
if (url.includes('article')) {
return await new Article(url).extract(html).then(async data => {
var _data$images, _data$product_name;
return {
...data,
images: !((_data$images = data.images) != null && _data$images.length) ? await googleClient.getImages((_data$product_name = data.product_name) != null ? _data$product_name : '') : data.images
};
});
}
if (url.includes('build.com')) {
return await new Builddotcom(url).extract(html).then(async data => {
var _data$images2, _data$product_name2;
return {
...data,
images: !((_data$images2 = data.images) != null && _data$images2.length) ? await googleClient.getImages((_data$product_name2 = data.product_name) != null ? _data$product_name2 : '') : data.images
};
});
}
if (url.includes('etsy')) {
return await new Etsy(url).extract(html).then(async data => {
var _data$images3, _data$product_name3;
return {
...data,
images: !((_data$images3 = data.images) != null && _data$images3.length) ? await googleClient.getImages((_data$product_name3 = data.product_name) != null ? _data$product_name3 : '') : data.images
};
});
}
if (url.includes('fineartamerica')) {
return await new Fineartamerica(url).extract(html).then(async data => {
var _data$images4, _data$product_name4;
return {
...data,
images: !((_data$images4 = data.images) != null && _data$images4.length) ? await googleClient.getImages((_data$product_name4 = data.product_name) != null ? _data$product_name4 : '') : data.images
};
});
}
if (url.includes('ikea')) {
return await new Ikea(url).extract(html).then(async data => {
var _data$images5, _data$product_name5;
return {
...data,
images: !((_data$images5 = data.images) != null && _data$images5.length) ? await googleClient.getImages((_data$product_name5 = data.product_name) != null ? _data$product_name5 : '') : data.images
};
});
}
if (url.includes('https://rugs.com')) {
return await new Rugsdotcom(url).extract(html).then(async data => {
var _data$images6, _data$product_name6;
return {
...data,
images: !((_data$images6 = data.images) != null && _data$images6.length) ? await googleClient.getImages((_data$product_name6 = data.product_name) != null ? _data$product_name6 : '') : data.images
};
});
}
if (url.includes('homedepot')) {
return await new Homedepot(url).extract(html).then(async data => {
var _data$images7, _data$product_name7;
return {
...data,
images: !((_data$images7 = data.images) != null && _data$images7.length) ? await googleClient.getImages(`${(_data$product_name7 = data.product_name) != null ? _data$product_name7 : ''} from homedepot`) : data.images
};
});
}
if (url.includes('zgallerie')) {
return await new Zgallerie(url).extract(html).then(async data => {
var _data$images8, _data$product_name8;
return {
...data,
images: !((_data$images8 = data.images) != null && _data$images8.length) ? await googleClient.getImages((_data$product_name8 = data.product_name) != null ? _data$product_name8 : '') : data.images
};
});
}
return await googleClient.extract(html);
} catch (error) {
logger.error('getRawData', error.message);
return null;
}
};
const getProductMetadata = (sourceUrl, html) => {
try {
const $ = load(html);
let schemaData = {};
try {
const jsonLdScripts = $('script[type="application/ld+json"]');
jsonLdScripts.each((_, element) => {
try {
const data = JSON.parse($(element).html() || '{}');
if (data['@type'] === 'Product' || data['@type'] === 'WebPage') {
schemaData = data;
}
if (Array.isArray(data)) {
data.forEach(item => {
if (item['@type'] === 'Product' || item['@type'] === 'WebPage') {
schemaData = item;
}
});
}
} catch (e) {
console.warn('Error parsing JSON-LD script:', e);
}
});
} catch (e) {
console.warn('Error extracting Schema.org data:', e);
}
const getMetaContent = (ogTag, twitterTag, schemaPath) => {
const ogContent = $(`meta[property="${ogTag}"]`).attr('content') || $(`meta[name="${ogTag}"]`).attr('content');
if (ogContent) return ogContent;
const twitterContent = $(`meta[name="${twitterTag}"]`).attr('content');
if (twitterContent) return twitterContent;
let schemaContent = schemaData;
for (const path of schemaPath) {
var _schemaContent;
schemaContent = (_schemaContent = schemaContent) == null ? void 0 : _schemaContent[path];
if (!schemaContent) break;
}
return typeof schemaContent === 'string' ? schemaContent : '';
};
const getNumericMetaContent = (ogTag, twitterTag, schemaPath) => {
const value = getMetaContent(ogTag, twitterTag, schemaPath);
return value ? parseFloat(value) : 0;
};
const images = [];
$('meta[property="og:image"]').each((_, element) => {
const imageUrl = $(element).attr('content');
if (imageUrl && !images.includes(imageUrl)) {
images.push(imageUrl);
}
});
if (images.length === 0) {
$('meta[name="twitter:image"]').each((_, element) => {
const imageUrl = $(element).attr('content');
if (imageUrl && !images.includes(imageUrl)) {
images.push(imageUrl);
}
});
}
if (images.length === 0 && schemaData.image) {
const schemaImages = Array.isArray(schemaData.image) ? schemaData.image : [schemaData.image];
schemaImages.forEach(img => {
if (img && !images.includes(img)) {
images.push(img);
}
});
}
const source = sourceUrl ? new URL(sourceUrl).hostname : '';
const priceString = getMetaContent('og:price:amount', 'twitter:price:amount', ['offers', 'price']) || getMetaContent('product:price:amount', 'twitter:price', ['price']);
const metadata = {
product_name: getMetaContent('og:title', 'twitter:title', ['name']),
product_url: getMetaContent('og:url', 'twitter:url', ['url']) || sourceUrl,
type: getMetaContent('og:type', 'twitter:card', ['@type']),
price: parseFloat(priceString) || 0,
height: getNumericMetaContent('product:height', 'twitter:height', ['height', 'value']),
width: getNumericMetaContent('product:width', 'twitter:width', ['width', 'value']),
depth: getNumericMetaContent('product:depth', 'twitter:depth', ['depth', 'value']),
tags: getMetaContent('article:tag', 'twitter:label', ['keywords']) || '',
images,
sku: getMetaContent('product:sku', 'twitter:data1', ['sku']),
source,
description: getMetaContent('og:description', 'twitter:description', ['description'])
};
return metadata;
} catch (error) {
console.error('Error processing content:', error);
throw error;
}
};
class UniversalPDPScrapper {
constructor(config) {
this.openAiClient = void 0;
this.scrape = async (url, timeoutInSeconds = 15) => {
try {
var _parsedOpenAiOutput$i, _partialScrapperOutpu, _partialScrapperOutpu2, _parsedOpenAiOutput$d, _parsedOpenAiOutput$t, _scrapperOutput$image, _scrapperOutput$image2;
const htmlContent = await getHtml(url);
const productMetadata = getProductMetadata(url, htmlContent);
const [partialScrapperOutput, parsedOpenAiOutput] = await Promise.all([getRawData(url, htmlContent), scrapeUsingAI(this.openAiClient, url, htmlContent, undefined, undefined, CONFIG.OPEN_AI.MODEL, 112000, timeoutInSeconds).catch(() => ({}))]);
removeNullsAndUndefined(partialScrapperOutput != null ? partialScrapperOutput : {});
removeNullsAndUndefined(parsedOpenAiOutput != null ? parsedOpenAiOutput : {});
removeNullsAndUndefined(productMetadata != null ? productMetadata : {});
const scrapperOutput = {
...parsedOpenAiOutput,
...productMetadata,
...partialScrapperOutput,
product_name: parsedOpenAiOutput.product_name || productMetadata.product_name || (partialScrapperOutput == null ? void 0 : partialScrapperOutput.product_name),
images: Array.from(new Set([...(productMetadata.images || []), ...((_parsedOpenAiOutput$i = parsedOpenAiOutput.images) != null ? _parsedOpenAiOutput$i : []), ...((_partialScrapperOutpu = partialScrapperOutput == null ? void 0 : partialScrapperOutput.images) != null ? _partialScrapperOutpu : [])])),
description: ((_partialScrapperOutpu2 = partialScrapperOutput == null ? void 0 : partialScrapperOutput.description) != null ? _partialScrapperOutpu2 : '').concat((_parsedOpenAiOutput$d = parsedOpenAiOutput.description) != null ? _parsedOpenAiOutput$d : ''),
tags: (_parsedOpenAiOutput$t = parsedOpenAiOutput.tags) == null ? void 0 : _parsedOpenAiOutput$t.toLowerCase(),
height: Number((parsedOpenAiOutput.height || (partialScrapperOutput == null ? void 0 : partialScrapperOutput.height) || productMetadata.height).toString().replace(/[^\d.]/g, '')).toFixed(2),
width: Number((parsedOpenAiOutput.width || (partialScrapperOutput == null ? void 0 : partialScrapperOutput.width) || productMetadata.width).toString().replace(/[^\d.]/g, '')).toFixed(2),
depth: Number((parsedOpenAiOutput.depth || (partialScrapperOutput == null ? void 0 : partialScrapperOutput.depth) || productMetadata.depth).toString().replace(/[^\d.]/g, '')).toFixed(2),
price: Number((parsedOpenAiOutput.price || (partialScrapperOutput == null ? void 0 : partialScrapperOutput.price) || productMetadata.price).toString().replace(/[^\d.]/g, '')).toFixed(2)
};
if (!scrapperOutput) {
throw new Error('Scrapper failed');
}
if (!scrapperOutput.type || scrapperOutput.type.toString() === 'product') {
var _determineType;
scrapperOutput.type = (_determineType = determineType(`${scrapperOutput.product_name} ${scrapperOutput.product_url}`)) != null ? _determineType : Types.PAINTING;
}
if (scrapperOutput.glbs) {
var _scrapperOutput$glbs$;
scrapperOutput.glb_to_use = (_scrapperOutput$glbs$ = scrapperOutput.glbs.find(glb => glb.includes('.glb'))) != null ? _scrapperOutput$glbs$ : scrapperOutput.glbs[0];
delete scrapperOutput.glbs;
}
scrapperOutput['supporting-surface'] = scrapperOutput.type === Types.PAINTING || scrapperOutput.type.includes('wall_') ? 'wall' : 'floor';
logger.info('scrape', scrapperOutput);
scrapperOutput.images = (_scrapperOutput$image = (_scrapperOutput$image2 = scrapperOutput.images) == null ? void 0 : _scrapperOutput$image2.slice(0, 5)) != null ? _scrapperOutput$image : [];
removeNullsAndUndefined(scrapperOutput);
return scrapperOutput;
} catch (err) {
logger.error('scrape', err.message);
return null;
}
};
if (config != null && config.openaiApiKey) {
CONFIG.OPEN_AI.API_KEY = config.openaiApiKey;
}
if (config != null && config.openaiOrgId) {
CONFIG.OPEN_AI.ORG_ID = config.openaiOrgId;
}
if (config != null && config.openaiModelId) {
CONFIG.OPEN_AI.MODEL = config.openaiModelId;
}
if (config && 'serpApiKey' in config && config.serpApiKey) {
CONFIG.SERP.API_KEY = config.serpApiKey;
}
if (config && 'googleApiKey' in config && config.googleApiKey) {
CONFIG.GOOGLE.API_KEY = config.googleApiKey;
}
if (config && 'googleCseId' in config && config.googleCseId) {
CONFIG.GOOGLE.CSE_ID = config.googleCseId;
}
this.openAiClient = getClient({
apiKey: CONFIG.OPEN_AI.API_KEY,
organization: CONFIG.OPEN_AI.ORG_ID
});
}
}
class Potterybarn {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
if (!html) {
if (!this.url.includes('potterybarn')) {
throw new Error('Invalid URL');
}
html = await getHtml(this.url);
}
const $ = load(html);
const res = {
product_name: $('#main-content > div.purchasing-container > div.pip-info > div > h1').text().replace(/[\r\n]/gm, '').trim(),
images: $('div.ossa8632-dummyheroimg > img').map((_i, el) => $(el).attr('src')).get(),
source: 'potterybarn',
product_url: this.url
};
return res;
}
}
class Wayfair {
constructor(url) {
this.url = void 0;
this.url = url;
}
async extract(html) {
var _dimensions$, _dimensions$2, _dimensions$3;
if (!html) {
if (!this.url.includes('wayfair')) {