web-scrapify/scrapers/ProductScraper.ts

A simple web scraper that can scrape product details from various e-commerce platforms.

import { RawAxiosRequestHeaders } from "axios";
import { ScraperHeaders } from "./Headers";
import { Product } from "./Product";
import { Logger } from "../utils/Logger";
import { URLBuilder } from "./URLBuilder";

export abstract class ProductScraper extends URLBuilder {
    protected baseUrl: string;
    protected url: string;
    protected headers: RawAxiosRequestHeaders;
    protected enableAgentRotations: boolean;
    protected timeout: number;
    protected logger: Logger;

    constructor({
        baseUrl, 
        enableLogging,
        enableAgentRotations,
        headers,
        timeout
    }: ScraperHeaders) {
        super();
        this.baseUrl = baseUrl;
        this.url = '';
        this.enableAgentRotations = enableAgentRotations;
        this.headers = headers;
        this.timeout = timeout;
        this.logger = new Logger(enableLogging);
    }

    abstract scrape(url: string): Promise<Product | null>;

    abstract scrapListings(url: string): Promise<Partial<Product>[]>;
}