n8n-nodes-duckduckgo-search
Version:
AI Agent-ready n8n community node for DuckDuckGo search. Search the web, images, news, and videos with no API key required and no outbound telemetry. Optionally fetch and extract the main text of result pages or any URL, and get DuckDuckGo Instant Answers
241 lines (240 loc) • 9.45 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.directWebSearch = directWebSearch;
exports.directImageSearch = directImageSearch;
exports.getSafeSearchString = getSafeSearchString;
const axios_1 = __importDefault(require("axios"));
const constants_1 = require("./constants");
function cleanText(text) {
return text
.replace(/ /g, ' ')
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'/g, "'")
.replace(/'/g, "'")
.replace(/<\/?b>/g, '')
.replace(/<[^>]*>/g, '')
.replace(/\s+/g, ' ')
.trim();
}
function isBlockedAdUrl(u) {
const host = u.hostname.toLowerCase();
if (host === 'duckduckgo.com' && u.pathname === '/y.js')
return true;
if ((host === 'bing.com' || host.endsWith('.bing.com')) && u.pathname === '/aclick')
return true;
if (u.searchParams.has('ad_provider') || u.searchParams.has('ad_type') || u.searchParams.has('ad_domain'))
return true;
return false;
}
function normaliseDdgUrl(raw) {
if (!raw)
return null;
const href = raw.startsWith('//') ? `https:${raw}` : raw;
if (!href.startsWith('http://') && !href.startsWith('https://'))
return null;
let u;
try {
u = new URL(href);
}
catch {
return null;
}
if (isBlockedAdUrl(u))
return null;
if (u.hostname.toLowerCase() === 'duckduckgo.com' && u.pathname === '/l/') {
const uddg = u.searchParams.get('uddg');
if (!uddg)
return null;
let target;
try {
target = new URL(uddg);
}
catch {
return null;
}
if (target.protocol !== 'http:' && target.protocol !== 'https:')
return null;
if (isBlockedAdUrl(target))
return null;
return uddg;
}
return href;
}
async function directWebSearch(query, options = {}) {
try {
const response = await axios_1.default.post('https://html.duckduckgo.com/html/', new URLSearchParams({
q: query,
b: '',
kl: options.locale || 'us-en',
kp: options.safeSearch === 'strict' ? '1' : options.safeSearch === 'moderate' ? '-1' : '-2',
}), {
headers: {
'User-Agent': constants_1.BROWSER_USER_AGENT,
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'no-cache',
'Pragma': 'no-cache',
},
timeout: 15000,
});
const results = [];
const html = response.data;
const resultSections = html.split(/<div[^>]*class="[^"]*result results_links[^"]*"[^>]*>/);
for (let i = 1; i < resultSections.length; i++) {
const section = resultSections[i];
const titleMatch = section.match(/<h2[^>]*class="result__title"[^>]*>[\s\S]*?<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/);
const snippetMatch = section.match(/<a[^>]*class="result__snippet"[^>]*href="[^"]*"[^>]*>(.*?)<\/a>/);
if (titleMatch) {
const rawUrl = titleMatch[1];
const title = titleMatch[2];
const description = snippetMatch ? snippetMatch[1] : '';
const url = normaliseDdgUrl(rawUrl);
if (title && url) {
results.push({
title: cleanText(title),
url: url,
description: cleanText(description),
});
if (options.maxResults && results.length >= options.maxResults) {
break;
}
}
}
}
if (results.length === 0 && resultSections.length === 1) {
if (response.status === 200 && html.length > 1000) {
throw new Error('DuckDuckGo web search response could not be parsed. ' +
'The page structure may have changed. Please try again later.');
}
}
return { results };
}
catch (error) {
console.error('Direct web search error:', error.message);
if (!error.code && !error.response) {
throw error;
}
if (error.code === 'ECONNABORTED') {
throw new Error('Web search request timed out. Please try again.');
}
else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') {
throw new Error('Unable to connect to DuckDuckGo. Please check your internet connection.');
}
else if (error.response && error.response.status === 429) {
throw new Error('Too many requests. Please wait a moment before trying again.');
}
else if (error.response && error.response.status >= 500) {
throw new Error('DuckDuckGo server error. Please try again later.');
}
else {
throw new Error(`Web search failed: ${error.message}`);
}
}
}
async function directImageSearch(query, options = {}, vqdHint) {
try {
const searchParams = new URLSearchParams({
q: query,
iax: 'images',
ia: 'images',
});
const searchUrl = `https://duckduckgo.com/?${searchParams.toString()}`;
let vqd;
if (vqdHint) {
vqd = vqdHint;
}
else {
const response = await axios_1.default.get(searchUrl, {
headers: {
'User-Agent': constants_1.BROWSER_USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
},
timeout: 15000,
});
const vqdMatch = response.data.match(/vqd=([\d-]+)/);
const extracted = vqdMatch ? vqdMatch[1] : null;
if (!extracted) {
throw new Error('DuckDuckGo image search token (VQD) could not be extracted. ' +
'Image search may be temporarily unavailable. Please try again later.');
}
vqd = extracted;
}
const imageParams = new URLSearchParams({
l: options.locale || 'us-en',
o: 'json',
q: query,
vqd: vqd,
f: ',,,,,',
p: options.safeSearch === 'strict' ? '1' : options.safeSearch === 'moderate' ? '-1' : '-2',
});
const imageResponse = await axios_1.default.get(`https://duckduckgo.com/i.js?${imageParams.toString()}`, {
headers: {
'User-Agent': constants_1.BROWSER_USER_AGENT,
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': searchUrl,
'X-Requested-With': 'XMLHttpRequest',
},
timeout: 15000,
});
const imageData = imageResponse.data;
const results = [];
if (imageData && imageData.results) {
for (const item of imageData.results) {
results.push({
title: item.title || '',
url: item.image || '',
thumbnail: item.thumbnail || '',
width: item.width,
height: item.height,
source: item.url || '',
});
if (options.maxResults && results.length >= options.maxResults) {
break;
}
}
}
return { results, vqd };
}
catch (error) {
console.error('Direct image search error:', error.message);
if (error.code === 'ECONNABORTED') {
throw new Error('Image search request timed out. Please try again.');
}
else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') {
throw new Error('Unable to connect to DuckDuckGo for image search. Please check your internet connection.');
}
else if (error.response && error.response.status === 429) {
throw new Error('Too many image search requests. Please wait a moment before trying again.');
}
else if (error.response && error.response.status === 403) {
throw new Error('DuckDuckGo image search returned 403 Forbidden. ' +
'The search token (VQD) may have expired or the request was blocked. Please try again.');
}
else {
throw new Error(`Image search failed: ${error.message}`);
}
}
}
function getSafeSearchString(value) {
switch (value) {
case 0:
return 'strict';
case -1:
return 'moderate';
case -2:
default:
return 'off';
}
}