universal-pdp-scrapper/dist/index.mjs.map

A universal pdp scrapper using cheerio & ChatGPT

{"version":3,"file":"index.mjs","sources":["../src/config/index.ts","../src/providers/log/index.ts","../src/utils/index.ts","../src/modules/openai.ts","../src/sources/article.ts","../src/sources/builddotcom.ts","../src/sources/etsy.ts","../src/sources/fineartamerica.ts","../src/sources/google.ts","../src/sources/homedepot.ts","../src/sources/ikea.ts","../src/sources/rugsdotcom.ts","../src/sources/zgallerie.ts","../src/types/scrapperOutput.ts","../src/client.ts","../src/sources/potterybarn.ts","../src/sources/wayfair.ts","../src/sources/westelm.ts"],"sourcesContent":["import { config } from 'dotenv';\n\nimport pkg from '../../package.json';\n\nconfig();\n\nexport default {\n  APP: {\n    NAME: pkg.name,\n    VERSION: pkg.version,\n    DESCRIPTION: pkg.description,\n    AUTHOR: pkg.author,\n    STAGE: (process.env.NODE_ENV === 'development' ? 'dev' : 'prod') as 'dev' | 'prod',\n    ENV: (process.env.NODE_ENV || 'development') as 'development' | 'production',\n  },\n  SERP: {\n    API_KEY: process.env.SERP_API_KEY || '',\n  },\n  OPEN_AI: {\n    API_KEY: process.env.OPEN_AI_SECRET_KEY ?? '',\n    ORG_ID: process.env.OPEN_AI_ORG_ID || undefined,\n    MODEL: process.env.OPEN_AI_MODEL || 'gpt-4o-mini',\n  },\n  GOOGLE: {\n    API_KEY: process.env.GOOGLE_API_KEY || '',\n    CSE_ID: process.env.GOOGLE_CSE_ID || '',\n  },\n};\n","/* eslint-disable no-console */\nexport class Logger {\n  constructor(private readonly name: string) {}\n\n  info(message: string, ...args: any) {\n    console.log(`[${this.name}] ${message}:`, ...args);\n  }\n\n  error(message: string, ...args: any) {\n    console.error(`[${this.name}] ${message}:`, ...args);\n  }\n\n  warn(message: string, ...args: any) {\n    console.warn(`[${this.name}] ${message}:`, ...args);\n  }\n\n  debug(message: string, ...args: any) {\n    console.debug(`[${this.name}] ${message}:`, ...args);\n  }\n\n  trace(message: string, ...args: any) {\n    console.trace(`[${this.name}] ${message}:`, ...args);\n  }\n}\n","/* eslint-disable no-param-reassign */\n/* eslint-disable no-restricted-syntax */\n\nimport axios from 'axios';\n\n// to always return type string event when s may be falsy other than empty-string\nexport const capitalize = (s: string) => {\n  if (!s) {\n    return 'Unnamed room';\n  }\n  // replace / with whitespace\n  s = s.replace(/\\//g, ' ');\n  // replace : with whitespace\n  s = s.replace(/:/g, ' ');\n  // replace , with whitespace\n  s = s.replace(/,/g, ' ');\n  // replace - with whitespace\n  s = s.replace(/-/g, ' ');\n  // replace _ with whitespace\n  s = s.replace(/_/g, ' ');\n  s = s.replaceAll('  ', ' ');\n  // capitalize first letter of each word\n  s = s.replace(/(^\\w{1})|(\\s+\\w{1})/g, (letter) => letter.toUpperCase());\n  // remove any numbers from the end\n  s = s.replace(/[\\d.]+$/, '');\n  return s.trim(); // trim trailing whitespace\n};\n\nexport function removeNullsAndUndefined(obj: Record<string, any>) {\n  for (const propName in obj) {\n    if (obj[propName] === null || obj[propName] === undefined || obj[propName] === '') {\n      delete obj[propName];\n    }\n  }\n}\n\ninterface ParseJSONOptions {\n  throwOnError?: boolean;\n  defaultValue?: any;\n}\n\n/**\n * Parses JSON from a markdown string or plain JSON string\n * @param mdString - The input string containing JSON (with or without markdown formatting)\n * @param options - Configuration options for parsing\n * @returns Parsed JSON object or default value if parsing fails\n * @throws Error if parsing fails and throwOnError is true\n * @examples\n ```typescript\n // With type safety\n interface User {\n   name: string;\n   age: number;\n }\n\n // Basic usage\n const user = parseJSONFromMarkdownString<User>(\\`\n   \\`\\`\\`json\n   {\n     \"name\": \"John\",\n     \"age\": 30\n   }\n   \\`\\`\\`\n \\`);\n\n // With options\n const userWithFallback = parseJSONFromMarkdownString<User>(\n   invalidJson,\n   {\n     throwOnError: false,\n     defaultValue: { name: 'Unknown', age: 0 }\n   }\n );\n\n // Direct JSON parsing\n const plainJsonUser = parseJSONFromMarkdownString<User>('{\"name\": \"John\", \"age\": 30}');\n ```\n */\nexport function parseJSONFromMarkdownString<T = any>(mdString: string, options: ParseJSONOptions = { throwOnError: true }): T {\n  try {\n    // Guard against invalid input\n    if (!mdString || typeof mdString !== 'string') {\n      throw new Error('Input must be a non-empty string');\n    }\n\n    // Normalize line endings\n    const normalizedString = mdString.replace(/\\r\\n/g, '\\n');\n\n    // Extract JSON content\n    let jsonString = normalizedString;\n    if (normalizedString.includes('```json')) {\n      // Find content between ```json and the next ```\n      const jsonBlockRegex = /```json\\s*([\\s\\S]*?)\\s*```/;\n      const matches = normalizedString.match(jsonBlockRegex);\n\n      if (!matches || !matches[1]) {\n        throw new Error('Invalid markdown JSON format');\n      }\n      [, jsonString] = matches;\n    }\n\n    // Clean up whitespace and try to parse\n    const trimmedString = jsonString.trim();\n    const parsed = JSON.parse(trimmedString);\n\n    // Validate that we got a value back\n    if (parsed === undefined) {\n      throw new Error('Parsing resulted in undefined value');\n    }\n\n    return parsed as T;\n  } catch (error) {\n    if (options.throwOnError) {\n      throw new Error(`Failed to parse JSON: ${(error as Error).message} => ${mdString}`);\n    }\n    return options.defaultValue as T;\n  }\n}\n\nexport async function getHtml(url: string) {\n  const parsedUrl = new URL(url);\n  let { data: html } = await axios\n    .get<string>(url, {\n      headers: {\n        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',\n        Accept: 'application/json, text/plain, */*',\n        'Accept-Encoding': 'gzip, deflate, br',\n        'Content-Type': 'application/json',\n        'X-Requested-With': 'XMLHttpRequest',\n        Origin: parsedUrl.origin,\n        Host: parsedUrl.host,\n        Connection: 'keep-alive',\n      },\n    })\n    .catch((err) => {\n      console.debug('getHtml.axios.get', err.message, err.response);\n      return { data: '<html><head><title></title></head></html>' };\n    });\n  if (typeof html !== 'string') {\n    html = JSON.stringify(html);\n  }\n  return html;\n}\n","import type { Tiktoken, TiktokenModel } from 'js-tiktoken';\nimport { encodingForModel } from 'js-tiktoken';\nimport { OpenAI } from 'openai';\n\nimport { Logger } from '@/providers/log';\nimport type { ProductMetadata } from '@/types/scrapperOutput';\nimport { parseJSONFromMarkdownString } from '@/utils';\n\nconst logger = new Logger('OpenAIService');\n\nexport const getClient = (config: { apiKey: string; organization?: string }) => new OpenAI(config);\n\n/**\n * @deprecated since version 1.2.0. Use {@link scrapeUsingAI} instead\n */\nexport const getCompletion = async (prompt: string, client: OpenAI, model = 'gpt-4o-mini') => {\n  try {\n    const result = await client.chat.completions.create({\n      model,\n      messages: [\n        {\n          role: 'system',\n          content: prompt,\n        },\n      ],\n      n: 1,\n      temperature: 0,\n    });\n\n    return result.choices[0]?.message?.content;\n  } catch (err) {\n    logger.error('getCompletion', (err as any).message);\n    return undefined;\n  }\n};\n\nconst trimToTokenLimit = (text: string, enc: Tiktoken, maxTokens = 128000, separator = '-------'): string => {\n  // Add input validation\n  if (!text || !enc) {\n    return text;\n  }\n\n  const tokens = enc.encode(text);\n\n  if (tokens.length <= maxTokens) {\n    return text;\n  }\n\n  if (text.includes(separator)) {\n    const parts = text.split(separator);\n    if (parts.length !== 3) {\n      // Handle invalid format more gracefully\n      return enc.decode(tokens.slice(0, maxTokens));\n    }\n\n    const [beforeHtml, html, afterHtml] = parts;\n    const beforeTokens = enc.encode(beforeHtml).length;\n    const afterTokens = enc.encode(afterHtml).length;\n    const intermediatoryTokens = enc.encode(`${separator}...[content trimmed]...${separator}`).length;\n\n    const availableTokens = maxTokens - beforeTokens - afterTokens - intermediatoryTokens;\n    if (availableTokens < 0) {\n      // Handle case where surrounding content is already too large\n      return enc.decode(tokens.slice(0, maxTokens));\n    }\n\n    const htmlTokens = enc.encode(html);\n    const halfAvailable = Math.floor(availableTokens / 2);\n    const startHtml = enc.decode(htmlTokens.slice(0, halfAvailable));\n    const endHtml = enc.decode(htmlTokens.slice(-halfAvailable));\n\n    return `${beforeHtml}${separator}${startHtml}...[content trimmed]...${endHtml}${separator}${afterHtml}`;\n  }\n\n  return enc.decode(tokens.slice(0, maxTokens));\n};\n\nexport const scrapeUsingAI = async (\n  client: OpenAI,\n  productUrl: string,\n  htmlContent?: string,\n  userId?: string,\n  metadata?: Record<string, string>,\n  model = 'gpt-4o-mini',\n  trimToMaxToken = 112000,\n  timeoutSeconds = 29,\n) => {\n  try {\n    const enc = encodingForModel(model as TiktokenModel);\n    const systemContext = `You are tasked with a job to accept from a given product URL and optionally its HTML content from the user and return the extracted information in a structured JSON format. You should not prompt the user to provide any additional details; rather, work with the details that are already provided to you.\n\nYou should be able to extract the following information:\n- **Product Name**\n- **Description**: A string describing the product in one line, such that when this string is searched in Google, it shows similar products.\n- **Tags**: A list of comma-separated words relating to the product.\n- **Product URL**\n- **Type**: One of the following: bathtub, bed, bench, bookshelf, cabinet, chair, coffee_table, console_table, day_bed, desk, dressing_table, faucet, floor_lamp, futon, light, loveseat, mattress, mirror, nightstand, painting, plant_stand, rug, side_table, sink, sleeper_sofa, sofa, sofa_bed, stool, storage_unit, study_table, table_lamp, toy, tv_stand, vanity, wall_lamp, wall_mirror, wall_shelf, water_closet.\n- **Price**: Converted to US dollars in format XXX.XX.\n- **Exact Height**: In inches and up to one decimal place, in XX.X format with unit converted to inches.\n- **Exact Width**: In inches and up to one decimal place, in XX.X format with unit converted to inches.\n- **Exact Depth**: In inches and up to one decimal place, in XX.X format with unit converted to inches.\n- **Images**: A list of image URLs.\n- **SKU**\n- **Source**: Brand name.\n\nThe information should be structured properly in the following JSON schema:\n\n{\n  \"product_name\": string,\n  \"product_url\": string,\n  \"type\": string,\n  \"price\": number,\n  \"height\": number,\n  \"width\": number,\n  \"depth\": number,\n  \"tags\": string,\n  \"images\": string[],\n  \"sku\": string,\n  \"source\": string,\n  \"description\": string\n}\n\n# Output Format\nStrictly return the extracted information in plain JSON format without any plain text and without any markdown formatting.\n\n# Examples\n\nExample JSON object for the product URL: [https://www.ikea.com/us/en/p/haegernaes-table-and-4-chairs-antique-stain-pine-70575947/]\n\n{\n  \"product_name\": \"HÄGERNÄS\",\n  \"product_url\": \"https://www.ikea.com/us/en/p/haegernaes-table-and-4-chairs-antique-stain-pine-70575947/\",\n  \"type\": \"table\",\n  \"price\": 199.99,\n  \"height\": 29.50,\n  \"width\": 59.10,\n  \"depth\": 35.40,\n  \"tags\": \"table, chairs, dining, antique, pine, furniture\",\n  \"images\": [\n    \"https://www.ikea.com/us/en/images/products/haegernaes-table-and-4-chairs-antique-stain-pine__1350925_pe951817_s5.jpg\"\n  ],\n  \"sku\": \"70575947\",\n  \"source\": \"ikea\",\n  \"description\": \"This sturdy dining set with a table and four chairs is perfect for your breakfast nook or smaller dining area. Solid pine is a natural material that ages beautifully and acquires its own unique character over time. Each table and chair has its own unique character due to the distinctive grain pattern. For a softer seat or to add a personal touch to the room, complete with a chair pad in the style and color of your choice.\"\n}`;\n    let userPrompt = `Here's the product URL: ${productUrl}${\n      htmlContent\n        ? `\\nand the HTML content:\n-------\n${htmlContent}\n-------\n`\n        : ''\n    }`.substring(0, 1048575);\n\n    const reservedTokens = enc.encode(systemContext).length;\n    const availableTokens = trimToMaxToken - reservedTokens;\n    userPrompt = trimToTokenLimit(userPrompt, enc, availableTokens);\n\n    // Create the main API call promise\n    const apiCallPromise = client.chat.completions.create({\n      model,\n      messages: [\n        {\n          role: 'system',\n          content: systemContext,\n        },\n        {\n          role: 'user',\n          content: userPrompt,\n        },\n      ],\n      n: 1,\n      temperature: 0,\n      response_format: {\n        type: 'json_object',\n      },\n      stream: false,\n      user: userId,\n      metadata,\n    });\n\n    // If timeout is provided, race between the API call and a timeout promise\n    if (timeoutSeconds) {\n      const timeoutPromise = new Promise((_, reject) => {\n        setTimeout(() => {\n          reject(new Error(`Operation timed out after ${timeoutSeconds} seconds`));\n        }, timeoutSeconds * 1000);\n      });\n\n      const result = (await Promise.race([apiCallPromise, timeoutPromise])) as OpenAI.Chat.Completions.ChatCompletion;\n      return parseJSONFromMarkdownString<ProductMetadata>(result.choices[0]?.message?.content || '');\n    }\n\n    // If no timeout provided, just execute the API call\n    const result = await apiCallPromise;\n    return parseJSONFromMarkdownString<ProductMetadata>(result.choices[0]?.message?.content || '');\n  } catch (error) {\n    logger.error('scrapeUsingAI', error instanceof Error ? error.message : 'Unknown error');\n    return {} as ProductMetadata;\n  }\n};\n","/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport class Article {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('article')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const $ = load(html!);\n    const dimension = $('#details > div > div.content.desktop.flex-grid > div:nth-child(3) > div > div')\n      .html()\n      ?.match(/[\\d.]+\"H x [\\d.]+\"W x [\\d.]+\"D/gm)?.[0];\n    const res: ScrapperOutput = {\n      product_name: $(\n        '#app > div > div.app-container-wrapper > div > div > div.product-nav > div.product-nav-desktop.container > div > div.primary-info > div > h2',\n      )\n        .text()\n        .replace(/[\\r\\n]/gm, '')\n        .trim(),\n      images: $(\n        '#app > div > div.app-container-wrapper > div > div > div.main-layout > div.product-buy-section-container.container.product-page-buy-section > div > div.product-buy-images > div.product-thumbnails-container > div > a.active.product-thumbnail > div > picture > img',\n      )\n        .map((_i, el) => $(el).attr('src')?.substring(0, $(el).attr('src')?.indexOf('?')))\n        .get(),\n      depth: dimension?.split('x')[2]?.trim()?.replace(/\\D+/g, ''),\n      width: dimension?.split('x')[1]?.trim()?.replace(/\\D+/g, ''),\n      height: dimension?.split('x')[0]?.trim()?.replace(/\\D+/g, ''),\n      price: $(\n        '#app > div > div.app-container-wrapper > div > div > div.product-nav > div.product-nav-desktop.container > div > div.product-navbar-desktop > div.sale-price > div > span',\n      )\n        .text()\n        ?.replace(/[\\r\\n]/gm, '')\n        ?.replace(/\\D+/g, '')\n        ?.trim(),\n      source: 'article',\n      product_url: this.url,\n    };\n    return res;\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport interface ProductJSON {\n  '@context': string;\n  '@type': string;\n  name: string;\n  description: string;\n  productID: string;\n  image: string;\n  sku: string;\n  brand: Brand;\n  offers: Offers;\n  aggregateRating: AggregateRating;\n}\n\nexport interface AggregateRating {\n  '@type': string;\n  ratingValue: number;\n  reviewCount: number;\n  worstRating: number;\n  bestRating: number;\n}\n\nexport interface Brand {\n  '@type': string;\n  name: string;\n}\n\nexport interface Offers {\n  '@type': string;\n  '@id': string;\n  priceCurrency: string;\n  price: number;\n  sku: string;\n  availability: string;\n  seller: Brand;\n  url: string;\n}\n\nexport class Builddotcom {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('build.com')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const $ = load(html!);\n    const productJson: ProductJSON = JSON.parse(\n      $('#main-content > div.center-ns.mw9-ns.b--theme-grey.bg-theme-white.ba-ns.pa2.pa3-ns.mt3-ns.mb3 > script').text(),\n    );\n    const thumbnail = productJson.image.replace('t_base,c_lpad,f_auto,dpr_auto,w_1200,h_1200', 't_base');\n    const productName = $('#pdp-buysection > div.cf.flex.db-ns.flex-column > div.fl.w-50-ns.order-0 > section > h1 > span.fw2.di-ns')\n      .text()\n      .replace(/[\\r\\n]/gm, '')\n      .trim();\n    const res: ScrapperOutput = {\n      product_name: productJson.name,\n      images: thumbnail ? [thumbnail] : [],\n      price: productJson.offers.price.toString(),\n      source: 'build.com',\n      sku: productJson.offers.sku,\n      product_url: productJson.offers.url,\n      artist: productJson.brand.name,\n      description: productName,\n      'supporting-surface': productJson.description.toLowerCase().includes('wall') ? 'wall' : 'floor',\n    };\n    return res;\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport class Etsy {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('etsy')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const $ = load(html!);\n    const res: ScrapperOutput = {\n      product_name: $('#listing-page-cart > div.wt-mb-xs-2 > h1')\n        .text()\n        .replace(/[\\r\\n]/gm, '')\n        .trim(),\n      images: $(\n        '#listing-right-column > div > div.body-wrap.wt-body-max-width.wt-display-flex-md.wt-flex-direction-column-xs > div.image-col.wt-order-xs-1.wt-mb-lg-6 > div > div > div.image-wrapper.wt-position-relative.carousel-container-responsive > div > div.image-carousel-container.wt-position-relative.wt-flex-xs-6.wt-order-xs-2.show-scrollable-thumbnails > ul > li > img',\n      )\n        .map((_i, el) => $(el).attr('data-src-zoom-image'))\n        .get(),\n      height: $('#product-details-content-toggle > div > ul > div > div:nth-last-child(1) > li > div')\n        .filter((_i, el) => $(el).text().includes('Height'))\n        .text()\n        ?.split(':')[1]\n        ?.trim()\n        ?.replace(/\\D+/g, ''),\n      width: $('#product-details-content-toggle > div > ul > div > div:nth-last-child(1) > li > div')\n        .filter((_i, el) => $(el).text().includes('Width'))\n        .text()\n        ?.split(':')[1]\n        ?.trim()\n        ?.replace(/\\D+/g, ''),\n      depth: $('#product-details-content-toggle > div > ul > div > div:nth-last-child(1) > li > div')\n        .filter((_i, el) => $(el).text().includes('Depth'))\n        .text()\n        ?.split(':')[1]\n        ?.trim()\n        ?.replace(/\\D+/g, ''),\n      material: $('#legacy-materials-product-details').text()?.split(':').pop()?.trim(),\n      price: $(\n        '#listing-page-cart > div.wt-mb-xs-6.wt-mb-lg-0 > div:nth-child(1) > div.wt-mb-xs-3 > div.wt-mb-xs-3 > div.wt-display-flex-xs.wt-align-items-center.wt-justify-content-space-between > div.wt-display-flex-xs.wt-align-items-center > p > span:nth-child(2)',\n      )\n        .text()\n        .replace(/[\\r\\n]/gm, '')\n        .trim(),\n      sku: this.url.split('/')[this.url.split('/').length - 2],\n      artist: $(\n        '#desktop_shop_owners_parent > div > div > div.wt-display-flex-xs.wt-align-items-center.wt-mb-xs-2 > div:nth-child(2) > p.wt-text-body-03.wt-line-height-tight.wt-mb-xs-1',\n      )\n        .text()\n        .trim(),\n      source: 'etsy',\n      product_url: this.url,\n    };\n    return res;\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport class Fineartamerica {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('fineartamerica')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const $ = load(html!);\n    const res: ScrapperOutput = {\n      product_name: $('#h1title').text().trim(),\n      images: [$('#productPreviewImage').attr('src')?.trim() ?? $('#mainimage').attr('src')?.trim()].filter((x) => !!x) as string[],\n      material: $('#priceDetailDiv > div:nth-child(2) > p:nth-child(2)').text().trim(),\n      price: `${$('#productCurrency').text().trim()} ${$('#productPrice').text().trim()}`,\n      artist: $('#artistName > a').text().trim(),\n      source: 'fineartamerica',\n      product_url: this.url,\n    };\n    return res;\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport { customsearch } from '@googleapis/customsearch';\nimport { load } from 'cheerio';\nimport { getJson } from 'serpapi';\n\nimport { Logger } from '../providers/log';\nimport type { ScrapperOutput } from '../types/scrapperOutput';\nimport { capitalize, getHtml } from '../utils';\n\nconst logger = new Logger('googleScrapper');\n\nexport class Google {\n  serpApiKey?: string;\n\n  googleApiKey: string = '';\n\n  googleCseId: string = '';\n\n  url: string;\n\n  constructor(\n    config: {\n      url: string;\n    } & (\n      | {\n          serpApiKey: string;\n        }\n      | {\n          googleApiKey: string;\n          googleCseId: string;\n        }\n    ),\n  ) {\n    this.url = config.url;\n    if ('serpApiKey' in config) {\n      this.serpApiKey = config.serpApiKey;\n    } else {\n      this.googleApiKey = config.googleApiKey;\n      this.googleCseId = config.googleCseId;\n    }\n  }\n\n  async getImages(prompt: string, useSerpApi?: boolean): Promise<string[]> {\n    const results: Record<string, any> | undefined = useSerpApi\n      ? await getJson('google', {\n          api_key: this.serpApiKey,\n          q: prompt,\n          gl: 'us',\n          hl: 'en',\n          tbm: 'isch',\n          filter: '1',\n          google_domain: 'google.com',\n          location: 'California, United States',\n        }).catch((err) => {\n          logger.error('getImages.usingSerpApi', err);\n          return undefined;\n        })\n      : await customsearch('v1')\n          .cse.list(\n            {\n              auth: this.googleApiKey,\n              q: prompt, // .replace(/\\s+/g, '+'),\n              cr: 'countryUS',\n              cx: this.googleCseId,\n              gl: 'us',\n              hl: 'en',\n              // lr: 'lang_en',\n              filter: '1',\n              num: 5,\n              safe: 'active',\n              searchType: 'image',\n            },\n            {\n              http2: true,\n            },\n          )\n          .catch((err) => {\n            logger.error('getImages.usingGoogleApi', err);\n            return undefined;\n          });\n    return useSerpApi\n      ? (results?.images_results.slice(0, 3) ?? []).map((image: any) => image.original)\n      : (results?.data.items?.map((item: any) => item.link!) ?? []);\n  }\n\n  async extract(html?: string) {\n    try {\n      if (!html) {\n        html = await getHtml(this.url);\n      }\n      const parsedURL = new URL(this.url);\n      const domain = parsedURL.hostname.replace('www.', '');\n      const domainParts = domain.split('.');\n      const pathName = parsedURL.pathname.replaceAll('/', ' ').trim();\n      const $ = load(html!);\n      const title = `${capitalize(`${pathName} ${$('head > title').text().trim()}`)}`\n        .toLowerCase()\n        .replace(/\\b(?![\\w\\d]+\\b)[^\\s]+\\b/g, '')\n        .replace(domainParts.at(-2) ?? '', '')\n        .replace(/\\s{2,}/g, ' ')\n        .split(' ')\n        .filter(\n          (word) =>\n            !(\n              word.includes('0') ||\n              word.includes('1') ||\n              word.includes('2') ||\n              word.includes('3') ||\n              word.includes('4') ||\n              word.includes('5') ||\n              word.includes('6') ||\n              word.includes('7') ||\n              word.includes('8') ||\n              word.includes('9') ||\n              word.includes('&') ||\n              word.includes('|') ||\n              word.includes('=')\n            ),\n        )\n        .join(' ')\n        .trim();\n      const processedTitle = Array.from(new Set(`${title} from ${domain}`.toLowerCase().split(' '))).join(' ');\n      const images = await this.getImages(processedTitle, !!this.serpApiKey);\n      if (!images.length && this.serpApiKey && this.googleApiKey && this.googleCseId) {\n        logger.warn('No images found, trying without serp api');\n        images.push(...(await this.getImages(processedTitle, false)));\n      }\n      const res: ScrapperOutput = {\n        images,\n        product_name: capitalize(title),\n        source: domainParts.at(-2),\n        product_url: this.url,\n      };\n      return res;\n    } catch (error) {\n      logger.error('extract', (error as any)?.message ?? error);\n      throw error;\n    }\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport interface HomedepotProductStructureData {\n  '@context': string;\n  '@type': string;\n  name: string;\n  image: string[];\n  description: string;\n  productID: string;\n  sku: string;\n  gtin13: string;\n  depth: string;\n  height: string;\n  width: string;\n  color: string;\n  weight: string;\n  brand: Brand;\n  aggregateRating: AggregateRating;\n  offers: Offers;\n  review: Review[];\n}\n\nexport interface AggregateRating {\n  '@type': string;\n  ratingValue: string;\n  reviewCount: number;\n}\n\nexport interface Brand {\n  '@type': BrandType;\n  name: string;\n}\n\nexport enum BrandType {\n  Brand = 'Brand',\n  Person = 'Person',\n}\n\nexport interface Offers {\n  '@type': string;\n  url: string;\n  priceCurrency: string;\n  price: number;\n  priceValidUntil: string;\n  availability: string;\n  hasMerchantReturnPolicy: HasMerchantReturnPolicy;\n}\n\nexport interface HasMerchantReturnPolicy {\n  '@type': string;\n  applicableCountry: string;\n  returnPolicyCategory: string;\n  merchantReturnDays: number;\n}\n\nexport interface Review {\n  '@type': ReviewType;\n  reviewRating: ReviewRating;\n  author: Brand;\n  headline: string;\n  reviewBody: string;\n}\n\nexport enum ReviewType {\n  Review = 'Review',\n}\n\nexport interface ReviewRating {\n  '@type': ReviewRatingType;\n  ratingValue: number;\n  bestRating: string;\n}\n\nexport enum ReviewRatingType {\n  Rating = 'Rating',\n}\n\nexport class Homedepot {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string): Promise<ScrapperOutput> {\n    if (!html) {\n      if (!this.url.includes('homedepot')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const $ = load(html!);\n    const stringifiedLdJson = $('#thd-helmet__script--productStructureData').html();\n    const ldJson: HomedepotProductStructureData = JSON.parse(stringifiedLdJson ?? '');\n    return {\n      product_name: ldJson.name,\n      images: ldJson.image.map((img) => img.replace('_100', '_1024')),\n      height: ldJson.height,\n      width: ldJson.width,\n      depth: ldJson.depth,\n      material: ldJson.color,\n      price: ldJson.offers.price.toString(),\n      sku: ldJson.sku,\n      artist: ldJson.brand.name,\n      product_url: this.url,\n      source: 'homedepot',\n      description: ldJson.description,\n      tags: 'furniture',\n    };\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport axios from 'axios';\nimport { load } from 'cheerio';\n\nimport { Logger } from '../providers/log';\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\n// Generated by https://quicktype.io\n\nexport interface IkeaProduct {\n  catalogRefs?: CatalogRefs;\n  currencyCode?: string;\n  experimental?: Experimental;\n  globalId?: string;\n  id?: string;\n  mainImage?: Image;\n  name?: string;\n  pipUrl?: string;\n  price?: string;\n  priceExclTax?: string;\n  priceExclTaxNumeral?: number;\n  priceNumeral?: number;\n  revampPrice?: RevampPrice;\n  styleGroup?: string;\n  typeName?: string;\n  validDesignText?: string;\n}\n\nexport interface CatalogRefs {\n  products: Products;\n  themes: Products;\n}\n\nexport interface Products {\n  elements?: Products[];\n  id: string;\n  name: string;\n  url: string;\n}\n\nexport interface Experimental {\n  contextualImage: Image;\n  isBreathTakingItem: boolean;\n  isFamilyPrice: boolean;\n  isNewLowerPrice: boolean;\n  isNewProduct: boolean;\n  isTimeRestricted: boolean;\n  priceUnit: string;\n  rating: Rating;\n  technicalCompliance: TechnicalCompliance;\n}\n\nexport interface Image {\n  alt: string;\n  id: string;\n  imageFileName: string;\n  type: string;\n  url: string;\n}\n\nexport interface Rating {\n  count: number;\n  enabled: boolean;\n  maxValue: number;\n  percentage: number;\n  value: number;\n}\n\nexport interface TechnicalCompliance {\n  valid: boolean;\n}\n\nexport interface RevampPrice {\n  currencyPrefix: string;\n  currencySuffix: string;\n  currencySuffixZeroDecimals: boolean;\n  currencySymbol: string;\n  decimals: string;\n  hasTrailingCurrency: boolean;\n  integer: string;\n  numDecimals: number;\n  separator: string;\n}\n\n// Generated by https://quicktype.io\n\nexport interface IkeaModel {\n  models: Model[];\n  itemType: string;\n  categorization: Categorization;\n  variations: Variation[];\n}\n\nexport interface Categorization {\n  hfbName: string;\n  hfbId: string;\n  rangeName: string;\n  rangeId: string;\n  areaName: string;\n  areaId: string;\n}\n\nexport interface Model {\n  markets: string[];\n  url: string;\n  geoEnabled: boolean;\n}\n\nexport interface Variation {\n  localId: string;\n  itemType: string;\n  models: Model[];\n}\n\nconst logger = new Logger('IkeaScrapper');\n\nexport class Ikea {\n  static readonly BASE_URL = 'https://www.ikea.com';\n\n  readonly url: string;\n\n  readonly sku: string;\n\n  constructor(url: string) {\n    this.url = url;\n    const { pathname } = new URL(this.url);\n    // remote trailing / from pathname\n    this.sku = pathname.slice(1).replace(/\\/$/, '').split('-').pop()!.replace(/[^\\d]/g, '');\n  }\n\n  async getProductDetails() {\n    try {\n      return await axios\n        .get<IkeaProduct>(`${Ikea.BASE_URL}/us/en/products/${this.sku.slice(-3)}/${this.sku}.json`)\n        .then((response) => response.data);\n    } catch (err) {\n      logger.error('getProductDetails', (err as any)?.message, (err as any).response?.status, (err as any).response?.data);\n      return null;\n    }\n  }\n\n  async getGLBs() {\n    try {\n      return await axios\n        .get<IkeaModel>(`${Ikea.BASE_URL}/global/assets/rotera/resources/${this.sku}.json`)\n        .then((response) => response.data);\n    } catch (err) {\n      logger.debug('getGLBs', (err as any)?.message, (err as any).response?.status, (err as any).response?.data);\n      return null;\n    }\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('ikea')) {\n        throw new Error('Invalid URL');\n      }\n      const response = await axios.get(this.url);\n      html = response.data;\n    }\n    const [productDetails, glbs] = await Promise.allSettled([this.getProductDetails(), this.getGLBs()]).then(\n      (resolvedPromises) =>\n        resolvedPromises.map((p) => {\n          if (p.status === 'fulfilled') {\n            return p.value;\n          }\n          return null;\n        }) as [IkeaProduct | null, IkeaModel | null],\n    );\n    let res: ScrapperOutput = {};\n    if (!productDetails || Object.keys(productDetails).length === 0) {\n      const $ = load(html!);\n      res = {\n        product_name: $(\n          '#pip-buy-module-content > div.pip-temp-price-module.pip-temp-price-module--informational.pip-temp-price-module--small.pip-temp-price-module--regular-price-package > div.pip-temp-price-module__information > div > h1 > span.pip-header-section__title--big.notranslate',\n        )\n          .text()\n          .replace(/[\\r\\n]/gm, '')\n          .trim(),\n        images: $('.pip-image')\n          .map((_i, el) => $(el).attr('src'))\n          .get(),\n        depth: $(\n          '#range-modal-mount-node > div > div:nth-child(3) > div > div.pip-sheets__content-wrapper > div > div > div > div.pip-product-dimensions__dimensions-container > p',\n        )\n          .filter((_i, el) => !!$(el).html()?.includes('Length'))\n          .text()\n          ?.split(' ')[0]\n          ?.trim()\n          ?.replace(/\\D+/g, ''),\n        width: $(\n          '#range-modal-mount-node > div > div:nth-child(3) > div > div.pip-sheets__content-wrapper > div > div > div > div.pip-product-dimensions__dimensions-container > p',\n        )\n          .filter((_i, el) => !!$(el).html()?.includes('Width'))\n          .text()\n          ?.split(' ')[0]\n          ?.trim()\n          ?.replace(/\\D+/g, ''),\n        height: $(\n          '#range-modal-mount-node > div > div:nth-child(3) > div > div.pip-sheets__content-wrapper > div > div > div > div.pip-product-dimensions__dimensions-container > p',\n        )\n          .filter((_i, el) => !!$(el).html()?.includes('height'))\n          .text()\n          ?.split(' ')[0]\n          ?.trim()\n          ?.replace(/\\D+/g, ''),\n        price: $(\n          '#pip-buy-module-content > div.pip-temp-price-module.pip-temp-price-module--informational.pip-temp-price-module--small.pip-temp-price-module--regular-price-package > div.pip-temp-price-module__price > div > span > span:nth-child(1) > span.pip-temp-price__integer',\n        )\n          .text()\n          .replace(/[\\r\\n]/gm, '')\n          .trim(),\n        sku: this.sku,\n        source: 'ikea',\n        product_url: this.url,\n        glbs:\n          glbs?.models.map((model: Model) => model.url) ??\n          Array.from(new Set(html!.match(/https:\\/\\/web-api\\.ikea\\.com\\/dimma\\/assets\\/.[^\"]*\\.glb/g))).sort(),\n      };\n    } else {\n      res = {\n        product_name: productDetails?.name,\n        images: productDetails?.mainImage?.url ? [productDetails.mainImage.url] : [],\n        price: productDetails?.priceNumeral?.toString(),\n        sku: this.sku,\n        product_url: this.url,\n        source: 'ikea',\n        glbs: glbs?.models.map((model: Model) => model.url) ?? [],\n      };\n    }\n    return res;\n  }\n}\n","/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport class Rugsdotcom {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('rugs.com')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const $ = load(html!);\n    const thumbnail = $('#FullScreenImageGallery > div:nth-child(3) > div > img').attr('src');\n    const res: ScrapperOutput = {\n      product_name: $(\n        '#react-root > div.row.product-display--content-container > div.col-12.col-md-6.pl-md-0.pr-xl-0.product-display--description > div:nth-child(2) > div > div:nth-child(1) > div > h1',\n      )\n        .text()\n        .replace(/[\\r\\n]/gm, '')\n        .trim(),\n      images: thumbnail ? [thumbnail.substring(0, thumbnail.indexOf('?'))] : [],\n      price: $(\n        '#add_to_cart_form > div:nth-child(1) > div > div > div.col-12.px-md-0.d-flex.align-items-center > span.h2-bold-no-margin.price.mr-2',\n      )\n        .text()\n        ?.replace(/[\\r\\n]/gm, '')\n        ?.replace(/\\D+/g, '')\n        ?.trim(),\n      source: 'rugs.com',\n      sku: $('#react-root > div.row.d-none.d-md-flex.py-3.product-display--breadcrumbs.width-fixed > div > strong').text()?.trim(),\n      product_url: this.url,\n    };\n    return res;\n  }\n}\n","/* eslint-disable no-param-reassign */\n\nimport { getHtml } from '@/utils';\n\nimport type { ScrapperOutput } from '../types/scrapperOutput';\n\nexport class Zgallerie {\n  readonly url: string;\n\n  constructor(url: string) {\n    this.url = url;\n  }\n\n  async extract(html?: string) {\n    if (!html) {\n      if (!this.url.includes('zgallerie')) {\n        throw new Error('Invalid URL');\n      }\n      html = await getHtml(this.url);\n    }\n    const re = /<script id=\"__NEXT_DATA__\" type=\"application\\/json\">(.+)<\\/script>/;\n    const result = html.match(re);\n    const info = JSON.parse(result?.[1]?.substring(0, result[1].indexOf('</script>')) ?? '');\n    const res: ScrapperOutput = {\n      product_name: info.props.pageProps.product.name.split('~')[0],\n      images: info.props.pageProps.product.images.map((image: any) => image.url_zoom),\n      height: info.props.pageProps.product.height?.toString(),\n      width: info.props.pageProps.product.width?.toString(),\n      depth: info.props.pageProps.product.depth?.toString(),\n      price: info.props.pageProps.product.calculated_price?.toString(),\n      sku: info.props.pageProps.product.sku?.toString(),\n      artist: info.props.pageProps.product.additional_info.artist_name ? info.props.pageProps.product.additional_info.artist_name[0] : '',\n      source: 'zgallerie',\n      product_url: this.url,\n    };\n    return res;\n  }\n}\n","export enum Types {\n  BATHTUB = 'bathtub',\n  BED = 'bed',\n  BENCH = 'bench',\n  BOOKSHELF = 'bookshelf',\n  CABINET = 'cabinet',\n  CHAIR = 'chair',\n  COFFEE_TABLE = 'coffee_table',\n  CONSOLE_TABLE = 'console_table',\n  DAY_BED = 'day_bed',\n  DESK = 'desk',\n  DRESSING_TABLE = 'dressing_table',\n  FAUCET = 'faucet',\n  FLOOR_LAMP = 'floor_lamp',\n  FUTON = 'futon',\n  LIGHT = 'light',\n  LOVESEAT = 'loveseat',\n  MATTRESS = 'mattress',\n  MIRROR = 'mirror',\n  NIGHTSTAND = 'nightstand',\n  PAINTING = 'painting',\n  PLANT_STAND = 'plant_stand',\n  RUG = 'rug',\n  SIDE_TABLE = 'side_table',\n  SINK = 'sink',\n  SLEEPER_SOFA = 'sleeper_sofa',\n  SOFA = 'sofa',\n  SOFA_BED = 'sofa_bed',\n  STOOL = 'stool',\n  STORAGE_UNIT = 'storage_unit',\n  STUDY_TABLE = 'study_table',\n  TABLE_LAMP = 'table_lamp',\n  TOY = 'toy',\n  TV_STAND = 'tv_stand',\n  VANITY = 'vanity',\n  WALL_LAMP = 'wall_lamp',\n  WALL_MIRROR = 'wall_mirror',\n  WALL_SHELF = 'wall_shelf',\n  WATER_CLOSET = 'water_closet',\n}\n\nexport interface ScrapperOutput {\n  product_name?: string;\n  images?: string[];\n  height?: string;\n  width?: string;\n  depth?: string;\n  material?: string;\n  price?: string;\n  sku?: string;\n  artist?: string;\n  type?: Types;\n  product_url?: string;\n  source?: string;\n  glbs?: string[];\n  glb_to_use?: string;\n  description?: string;\n  tags?: string;\n  'supporting-surface'?: 'floor' | 'wall';\n}\n\nexport interface ProductMetadata {\n  product_name: string;\n  product_url: string;\n  type: string;\n  price: number;\n  height: number;\n  width: number;\n  depth: number;\n  tags: string;\n  images: string[];\n  sku: string;\n  source: string;\n  description: string;\n}\n\nexport interface ScraperInput {\n  url: string;\n  html: string;\n}\n","/* eslint-disable no-restricted-syntax */\n/* eslint-disable class-methods-use-this */\n/* eslint-disable no-param-reassign */\nimport { load } from 'cheerio';\nimport type { OpenAI } from 'openai';\n\nimport CONFIG from './config';\nimport { getClient, scrapeUsingAI } from './modules/openai';\nimport { Logger } from './providers/log';\nimport { Article } from './sources/article';\nimport { Builddotcom } from './sources/builddotcom';\nimport { Etsy } from './sources/etsy';\nimport { Fineartamerica } from './sources/fineartamerica';\nimport { Google } from './sources/google';\nimport { Homedepot } from './sources/homedepot';\nimport { Ikea } from './sources/ikea';\nimport { Rugsdotcom } from './sources/rugsdotcom';\nimport { Zgallerie } from './sources/zgallerie';\nimport type { ProductMetadata, ScrapperOutput } from './types/scrapperOutput';\nimport { Types } from './types/scrapperOutput';\nimport { capitalize, getHtml, removeNullsAndUndefined } from './utils';\n\nconst logger = new Logger('ScrapperClient');\n\nconst determineType = (searchString: string) => {\n  try {\n    const allTags = capitalize(searchString).toLocaleLowerCase();\n    const allModelTypes = Object.values(Types);\n    const determinedType = allModelTypes.find((type) => {\n      const typeTags = type.toLocaleLowerCase().split('_');\n      return typeTags.some((tag) => allTags.includes(tag));\n    });\n    return determinedType as Types | undefined;\n  } catch (err) {\n    logger.error('determineType', (err as any).message);\n    return undefined;\n  }\n};\n\nconst isAnImage = (html: string): boolean => {\n  return !html.includes('<html');\n};\n\nexport const getRawData = async (url: string, html: string) => {\n  try {\n    if (isAnImage(html)) {\n      return {\n        images: [url],\n        product_url: url,\n        product_name: url.split('/').pop()?.split('.').shift(),\n        source: 'user-sourced',\n      };\n    }\n    const googleClient = new Google({\n      url,\n      serpApiKey: CONFIG.SERP.API_KEY,\n      googleApiKey: CONFIG.GOOGLE.API_KEY,\n      googleCseId: CONFIG.GOOGLE.CSE_ID,\n    });\n    if (url.includes('article')) {\n      return await new Article(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    if (url.includes('build.com')) {\n      return await new Builddotcom(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    if (url.includes('etsy')) {\n      return await new Etsy(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    if (url.includes('fineartamerica')) {\n      return await new Fineartamerica(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    if (url.includes('ikea')) {\n      return await new Ikea(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    // if (url.includes('potterybarn')) {\n    //   return await new Potterybarn(url).extract(html);\n    // }\n    if (url.includes('https://rugs.com')) {\n      return await new Rugsdotcom(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    // if (url.includes('westelm')) {\n    //   return await new Westelm(url).extract(html);\n    // }\n    if (url.includes('homedepot')) {\n      return await new Homedepot(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(`${data.product_name ?? ''} from homedepot`) : data.images,\n        };\n      });\n    }\n    if (url.includes('zgallerie')) {\n      return await new Zgallerie(url).extract(html).then(async (data) => {\n        return {\n          ...data,\n          images: !data.images?.length ? await googleClient.getImages(data.product_name ?? '') : data.images,\n        };\n      });\n    }\n    return await googleClient.extract(html);\n  } catch (error) {\n    logger.error('getRawData', (error as any).message);\n    return null;\n  }\n};\n\nexport const getProductMetadata = (sourceUrl: string, html: string): ProductMetadata => {\n  try {\n    const $ = load(html);\n\n    // Extract Schema.org JSON-LD data\n    let schemaData: any = {};\n    try {\n      const jsonLdScripts = $('script[type=\"application/ld+json\"]');\n      jsonLdScripts.each((_, element) => {\n        try {\n          const data = JSON.parse($(element).html() || '{}');\n          // Look for Product or WebPage type schemas\n          if (data['@type'] === 'Product' || data['@type'] === 'WebPage') {\n            schemaData = data;\n          }\n          // Handle array of schemas\n          if (Array.isArray(data)) {\n            data.forEach((item) => {\n              if (item['@type'] === 'Product' || item['@type'] === 'WebPage') {\n                schemaData = item;\n              }\n            });\n          }\n        } catch (e) {\n          console.warn('Error parsing JSON-LD script:', e);\n        }\n      });\n    } catch (e) {\n      console.warn('Error extracting Schema.org data:', e);\n    }\n\n    // Helper function to get meta content with multiple fallbacks\n    const getMetaContent = (ogTag: string, twitterTag: string, schemaPath: string[]): string => {\n      // Try OpenGraph\n      const ogContent = $(`meta[property=\"${ogTag}\"]`).attr('content') || $(`meta[name=\"${ogTag}\"]`).attr('content');\n      if (ogContent) return ogContent;\n\n      // Try Twitter\n      const twitterContent = $(`meta[name=\"${twitterTag}\"]`).attr('content');\n      if (twitterContent) return twitterContent;\n\n      // Try Schema.org\n      let schemaContent = schemaData;\n      for (const path of schemaPath) {\n        schemaContent = schemaContent?.[path];\n        if (!schemaContent) break;\n      }\n      return typeof schemaContent === 'string' ? schemaContent : '';\n    };\n\n    // Helper function to get numeric content with fallbacks\n    const getNumericMetaContent = (ogTag: string, twitterTag: string, schemaPath: string[]): number => {\n      const value = getMetaContent(ogTag, twitterTag, schemaPath);\n      return value ? parseFloat(value) : 0;\n    };\n\n    // Initialize and populate images array with fallbacks\n    const images: string[] = [];\n\n    // Try OG images\n    $('meta[property=\"og:image\"]').each((_, element) => {\n      const imageUrl = $(element).attr('content');\n      if (imageUrl && !images.includes(imageUrl)) {\n        images.push(imageUrl);\n      }\n    });\n\n    // If no OG images, try Twitter images\n    if (images.length === 0) {\n      $('meta[name=\"twitter:image\"]').each((_, element) => {\n        const imageUrl = $(element).attr('content');\n        if (imageUrl && !images.includes(imageUrl)) {\n          images.push(imageUrl);\n        }\n      });\n    }\n\n    // If still no images, try Schema.org images\n    if (images.length === 0 && schemaData.image) {\n      const schemaImages = Array.isArray(schemaData.image) ? schemaData.image : [schemaData.image];\n\n      schemaImages.forEach((img: string) => {\n        if (img && !images.includes(img)) {\n          images.push(img);\n        }\n      });\n    }\n\n    // Get source domain from URL if available\n    const source = sourceUrl ? new URL(sourceUrl).hostname : '';\n\n    // Price extraction with Schema.org fallback\n    const priceString =\n      getMetaContent('og:price:amount', 'twitter:price:amount', ['offers', 'price']) ||\n      getMetaContent('product:price:amount', 'twitter:price', ['price']);\n\n    // Create the metadata object with all fallbacks\n    const metadata: ProductMetadata = {\n      product_name: getMetaContent('og:title', 'twitter:title', ['name']),\n      product_url: getMetaContent('og:url', 'twitter:url', ['url']) || sourceUrl,\n      type: getMetaContent('og:type', 'twitter:card', ['@type']),\n      price: parseFloat(priceString) || 0,\n      height: getNumericMetaContent('product:height', 'twitter:height', ['height', 'value']),\n      width: getNumericMetaContent('product:width', 'twitter:width', ['width', 'value']),\n      depth: getNumericMetaContent('product:depth', 'twitter:depth', ['depth', 'value']),\n      tags: getMetaContent('article:tag', 'twitter:label', ['keywords']) || '',\n      images,\n      sku: getMetaContent('product:sku', 'twitter:data1', ['sku']),\n      source,\n      description: getMetaContent('og:description', 'twitter:description', ['description']),\n    };\n\n    return metadata;\n  } catch (error) {\n    console.er