ddg-bulk-image-downloader/lib/downloader.js

Lazy way to download images from Duck Duck Go search results in bulk

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadImages = void 0;
const node_fetch_1 = require("node-fetch");
const download_image_1 = require("./download-image");
const path = require("path");
const fs_extra_1 = require("fs-extra");
const cli_ux_1 = require("cli-ux");
const utils_1 = require("./utils");
const filterOptionsToParam_1 = require("./filterOptionsToParam");
const ROOT_URL = "https://duckduckgo.com";
async function getToken(query) {
    try {
        const response = await node_fetch_1.default(`${ROOT_URL}/?q=${encodeURIComponent(query)}`).then((t) => t.text());
        const regex = /vqd='(.*?)'/gm;
        const payload = regex.exec(response);
        // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
        // @ts-ignore
        return payload[1];
    }
    catch (error) {
        throw new Error("Failed to get token!" + error);
    }
}
async function downloadImages({ query, limit, filter = () => true, imageOptions, outputPath = "", debug, }) {
    cli_ux_1.default.action.start("Initializing download", "", { stdout: true });
    const token = await getToken(query);
    const tokenQueryParamObj = { vqd: token };
    const queryParamString = utils_1.objToQueryString(Object.assign({ o: "json", l: "wt-wt", f: filterOptionsToParam_1.filterOptionsToParam(imageOptions), q: encodeURIComponent(query) }, tokenQueryParamObj));
    const url = `${ROOT_URL}/i.js?${queryParamString}`;
    let response;
    // Ensure output folder is created
    fs_extra_1.ensureDirSync(outputPath);
    let count = 0;
    let page = 1;
    const failed = [];
    while (count - failed.length < limit) {
        // console.log("Next:", response.next);
        let nextUrl = url;
        if (response === null || response === void 0 ? void 0 : response.next) {
            ++page;
            if (debug) {
                console.log("Going to page", page);
            }
            nextUrl = `${ROOT_URL}/${response.next}&${utils_1.objToQueryString(tokenQueryParamObj)}`;
        }
        console.log("Fetching", nextUrl);
        response = (await node_fetch_1.default(nextUrl)
            .then((t) => {
            console.log("Status", t.status);
            return t.json();
        })
            .catch((e) => {
            console.error(e);
            return undefined;
        }));
        const effectiveLimit = limit - (count - failed.length);
        const filteredImage = response.results.filter(filter);
        const toBeDownloadedImages = filteredImage.slice(0, Math.min(effectiveLimit, filteredImage.length));
        cli_ux_1.default.action.start(`Downloading ${toBeDownloadedImages.length} images from`, "page :" + page.toString(), { stdout: true });
        // Method 1 : Doesn't uses parallel downloadImage capabilities, but ensures all files present.
        // for (let i = 0; i < filteredImage.length; i++) {
        //   try {
        //     await downloadImage(filteredImage[i].image, `${outputPath + query}_${count++}`)
        //   } catch (error) {
        //     count--
        //     console.error('FFFF', count)
        //   }
        // }
        // count += filteredImage.length
        // Method 2 : Could leave holes, when URl returns non success. But fast.
        await Promise.all(toBeDownloadedImages.map(async (item) => {
            try {
                const savePath = path.join(outputPath, `${query}_${count++}`);
                // console.log(savePath)
                await download_image_1.default(item.image, savePath);
                // console.log("Success", count)
            }
            catch (error) {
                if (debug) {
                    console.error(error.message);
                }
                failed.push(error);
            }
            finally {
                // eslint-disable-next-line no-unsafe-finally
                return Promise.resolve();
            }
        }));
    }
    cli_ux_1.default.action.stop("done");
}
exports.downloadImages = downloadImages;