n8n-nodes-duckduckgo-search/dist/nodes/DuckDuckGo/htmlParser.js

AI Agent-ready n8n community node for DuckDuckGo search. Search the web, images, news, and videos with no API key required and no outbound telemetry. Optionally fetch and extract the main text of result pages or any URL, and get DuckDuckGo Instant Answers

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.parseHtmlResults = parseHtmlResults;
function parseHtmlResults(html) {
    const results = [];
    const resultRegex = /<div[^>]*class="[^"]*result[^"]*"[^>]*>([\s\S]*?)<\/div>/gi;
    let match;
    while ((match = resultRegex.exec(html)) !== null) {
        const resultDiv = match[1];
        const titleRegex = /<a[^>]*class="[^"]*result__a[^"]*"[^>]*(?:href="([^"]*)"[^>]*)?>(.*?)<\/a>/i;
        const titleMatch = titleRegex.exec(resultDiv);
        if (!titleMatch)
            continue;
        const title = titleMatch[2] ? titleMatch[2].replace(/<[^>]*>/g, '').trim() : '';
        const url = titleMatch[1] || '';
        let snippet = '';
        const snippetARegex = /<a[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)<\/a>/i;
        const snippetAMatch = snippetARegex.exec(resultDiv);
        if (snippetAMatch) {
            snippet = snippetAMatch[1].replace(/<[^>]*>/g, '').trim();
        }
        else {
            const snippetDivRegex = /<div[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)<\/div>/i;
            const snippetDivMatch = snippetDivRegex.exec(resultDiv);
            if (snippetDivMatch) {
                snippet = snippetDivMatch[1].replace(/<[^>]*>/g, '').trim();
            }
        }
        if (title && url && title.length > 0 && url.length > 0) {
            results.push({
                title: cleanHtml(title),
                url,
                snippet: cleanHtml(snippet),
                source: 'html',
            });
        }
    }
    return results;
}
function cleanHtml(text) {
    return text
        .replace(/&nbsp;/g, ' ')
        .replace(/&amp;/g, '&')
        .replace(/&lt;/g, '<')
        .replace(/&gt;/g, '>')
        .replace(/&quot;/g, '"')
        .replace(/&#39;/g, "'")
        .replace(/\s+/g, ' ')
        .trim();
}