n8n-nodes-duckduckgo-search/dist/nodes/DuckDuckGo/fallbackSearch.js

AI Agent-ready n8n community node for DuckDuckGo search. Search the web, images, news, and videos with no API key required and no outbound telemetry. Optionally fetch and extract the main text of result pages or any URL, and get DuckDuckGo Instant Answers

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.fallbackWebSearch = fallbackWebSearch;
exports.fallbackNewsSearch = fallbackNewsSearch;
exports.fallbackVideoSearch = fallbackVideoSearch;
const axios_1 = __importDefault(require("axios"));
const constants_1 = require("./constants");
function parseSearchResultsFromHTML(html) {
    const results = [];
    const resultRegex = /<div[^>]*class="[^"]*result[^"]*"[^>]*>([\s\S]*?)<\/div>/gi;
    const titleRegex = /<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/i;
    const snippetRegex = /<[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)<\/[^>]*>/i;
    let match;
    while ((match = resultRegex.exec(html)) !== null) {
        const resultHTML = match[1];
        const titleMatch = titleRegex.exec(resultHTML);
        const snippetMatch = snippetRegex.exec(resultHTML);
        if (titleMatch) {
            const rawHref = titleMatch[1];
            const title = titleMatch[2].replace(/<[^>]*>/g, '').trim();
            const body = snippetMatch ? snippetMatch[1].replace(/<[^>]*>/g, '').trim() : '';
            const normHref = normaliseDdgUrl(rawHref);
            if (title && normHref) {
                results.push({
                    title: cleanText(title),
                    href: normHref,
                    body: cleanText(body),
                });
            }
        }
    }
    return results;
}
function cleanText(text) {
    return text
        .replace(/&nbsp;/g, ' ')
        .replace(/&amp;/g, '&')
        .replace(/&lt;/g, '<')
        .replace(/&gt;/g, '>')
        .replace(/&quot;/g, '"')
        .replace(/&#39;/g, "'")
        .replace(/\s+/g, ' ')
        .trim();
}
function isBlockedAdUrl(u) {
    const host = u.hostname.toLowerCase();
    if (host === 'duckduckgo.com' && u.pathname === '/y.js')
        return true;
    if ((host === 'bing.com' || host.endsWith('.bing.com')) && u.pathname === '/aclick')
        return true;
    if (u.searchParams.has('ad_provider') || u.searchParams.has('ad_type') || u.searchParams.has('ad_domain'))
        return true;
    return false;
}
function normaliseDdgUrl(raw) {
    if (!raw)
        return null;
    const href = raw.startsWith('//') ? `https:${raw}` : raw;
    if (!href.startsWith('http://') && !href.startsWith('https://'))
        return null;
    let u;
    try {
        u = new URL(href);
    }
    catch {
        return null;
    }
    if (isBlockedAdUrl(u))
        return null;
    if (u.hostname.toLowerCase() === 'duckduckgo.com' && u.pathname === '/l/') {
        const uddg = u.searchParams.get('uddg');
        if (!uddg)
            return null;
        let target;
        try {
            target = new URL(uddg);
        }
        catch {
            return null;
        }
        if (target.protocol !== 'http:' && target.protocol !== 'https:')
            return null;
        if (isBlockedAdUrl(target))
            return null;
        return uddg;
    }
    return href;
}
async function fallbackWebSearch(query, options = {}) {
    try {
        const searchUrl = 'https://html.duckduckgo.com/html/';
        const params = new URLSearchParams({
            q: query,
            kl: options.locale || 'us-en',
            s: String(options.safeSearch || 'moderate'),
            df: options.time || '',
        });
        const response = await axios_1.default.get(`${searchUrl}?${params.toString()}`, {
            headers: {
                'User-Agent': constants_1.BROWSER_USER_AGENT,
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.5',
                'Accept-Encoding': 'gzip, deflate, br',
                'DNT': '1',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1',
            },
            timeout: 10000,
        });
        const results = parseSearchResultsFromHTML(response.data);
        const noResultsIndicator = response.data.includes('no-results') || results.length === 0;
        return {
            success: true,
            noResults: noResultsIndicator,
            results,
        };
    }
    catch (error) {
        console.error('Fallback search error:', error);
        return {
            success: false,
            noResults: true,
            results: [],
            error: `Fallback search failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
        };
    }
}
async function fallbackNewsSearch(query, options = {}) {
    try {
        const newsQuery = `${query} news`;
        return await fallbackWebSearch(newsQuery, options);
    }
    catch (error) {
        console.error('Fallback news search error:', error);
        return {
            success: false,
            noResults: true,
            results: [],
            error: `Fallback news search failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
        };
    }
}
async function fallbackVideoSearch(query, options = {}) {
    try {
        const videoQuery = `${query} site:youtube.com OR site:vimeo.com OR site:dailymotion.com`;
        return await fallbackWebSearch(videoQuery, options);
    }
    catch (error) {
        console.error('Fallback video search error:', error);
        return {
            success: false,
            noResults: true,
            results: [],
            error: `Fallback video search failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
        };
    }
}