n8n-nodes-duckduckgo-search
Version:
A powerful and comprehensive n8n community node that seamlessly integrates DuckDuckGo search capabilities into your workflows. Search the web, find images, discover news, and explore videos - all with privacy-focused, reliable results.
315 lines (314 loc) • 13.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.multiBackendSearch = exports.MultiBackendDuckDuckGoSearch = exports.Backend = void 0;
const axios_1 = __importDefault(require("axios"));
const duck_duck_scrape_1 = require("duck-duck-scrape");
const apiClient_1 = require("./apiClient");
var Backend;
(function (Backend) {
Backend["AUTO"] = "auto";
Backend["DUCK_DUCK_SCRAPE"] = "duck-duck-scrape";
Backend["SEARCH_API"] = "search-api";
Backend["HTML"] = "html";
Backend["LITE"] = "lite";
})(Backend = exports.Backend || (exports.Backend = {}));
class MultiBackendDuckDuckGoSearch {
constructor(backends = [Backend.AUTO]) {
this.cache = new Set();
if (backends.includes(Backend.AUTO)) {
this.backends = this.shuffleArray([
Backend.DUCK_DUCK_SCRAPE,
Backend.SEARCH_API,
Backend.HTML,
Backend.LITE
]);
}
else {
this.backends = backends;
}
}
shuffleArray(array) {
const shuffled = [...array];
for (let i = shuffled.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
}
return shuffled;
}
async searchWithDuckDuckScrape(query, options) {
try {
const result = await (0, duck_duck_scrape_1.search)(query, options);
if (result && result.results && result.results.length > 0) {
const convertedResults = result.results.map((item) => ({
title: item.title || '',
href: item.url || '',
body: item.description || '',
hostname: this.extractHostname(item.url || ''),
snippet: item.description || ''
}));
return {
success: true,
results: convertedResults.filter((r) => !this.cache.has(r.href) && (this.cache.add(r.href), true)),
backend: Backend.DUCK_DUCK_SCRAPE,
vqd: result.vqd,
timestamp: Date.now()
};
}
return {
success: true,
noResults: true,
results: [],
backend: Backend.DUCK_DUCK_SCRAPE,
timestamp: Date.now()
};
}
catch (error) {
throw new Error(`Duck-duck-scrape backend failed: ${error.message}`);
}
}
async searchWithSearchAPI(query, options) {
try {
const result = await (0, apiClient_1.searchWithAPI)(query, options);
if (result && result.results && result.results.length > 0) {
const convertedResults = result.results.map((item) => ({
title: item.title || '',
href: item.url || '',
body: item.description || item.body || '',
hostname: this.extractHostname(item.url || ''),
snippet: item.description || item.body || ''
}));
return {
success: true,
results: convertedResults.filter((r) => !this.cache.has(r.href) && (this.cache.add(r.href), true)),
backend: Backend.SEARCH_API,
vqd: result.vqd,
timestamp: Date.now(),
knowledge_graph: result.knowledge_graph,
ai_overview: result.ai_overview,
top_stories: result.top_stories,
related_searches: result.related_searches,
inline_images: result.inline_images,
inline_videos: result.inline_videos
};
}
return {
success: true,
noResults: true,
results: [],
backend: Backend.SEARCH_API,
timestamp: Date.now()
};
}
catch (error) {
throw new Error(`SearchAPI backend failed: ${error.message}`);
}
}
normalizeUrl(url) {
if (!url)
return '';
if (url.startsWith('http://') || url.startsWith('https://')) {
return url;
}
return `https://${url}`;
}
extractHostname(url) {
try {
return new URL(url).hostname;
}
catch {
return '';
}
}
async autoSearch(query, options) {
let lastError = null;
for (const backend of this.backends) {
try {
let result;
switch (backend) {
case Backend.HTML:
result = await this.searchWithHtmlBackend(query, options);
break;
case Backend.LITE:
result = await this.searchWithLiteBackend(query, options);
break;
case Backend.DUCK_DUCK_SCRAPE:
result = await this.searchWithDuckDuckScrape(query, options);
break;
case Backend.SEARCH_API:
result = await this.searchWithSearchAPI(query, options);
break;
default:
continue;
}
if (result.success && result.results.length > 0) {
return result;
}
}
catch (error) {
console.warn(`Backend ${backend} failed: ${error.message}`);
lastError = error;
continue;
}
}
return {
success: false,
noResults: true,
results: [],
error: `All backends failed. Last error: ${(lastError === null || lastError === void 0 ? void 0 : lastError.message) || 'Unknown error'}`,
backend: Backend.AUTO,
timestamp: Date.now()
};
}
async search(query, options = {}) {
const { backend = Backend.AUTO, ...searchOptions } = options;
switch (backend) {
case Backend.AUTO:
return await this.autoSearch(query, searchOptions);
case Backend.HTML:
return await this.searchWithHtmlBackend(query, searchOptions);
case Backend.LITE:
return await this.searchWithLiteBackend(query, searchOptions);
case Backend.DUCK_DUCK_SCRAPE:
return await this.searchWithDuckDuckScrape(query, searchOptions);
case Backend.SEARCH_API:
return await this.searchWithSearchAPI(query, searchOptions);
default:
throw new Error(`Unknown backend: ${backend}`);
}
}
async searchWithHtmlBackend(query, options = {}) {
var _a;
try {
const searchUrl = 'https://html.duckduckgo.com/html';
const response = await axios_1.default.post(searchUrl, new URLSearchParams({
q: query,
kl: options.locale || 'en-us',
df: options.time || '',
s: ((_a = options.safeSearch) === null || _a === void 0 ? void 0 : _a.toString()) || '1'
}), {
headers: {
'Referer': 'https://html.duckduckgo.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded'
},
timeout: 15000
});
const results = this.parseHtmlResults(response.data);
return {
results: results.filter(r => !this.cache.has(r.href) && (this.cache.add(r.href), true)),
backend: Backend.HTML,
success: true,
timestamp: Date.now()
};
}
catch (error) {
throw new Error(`HTML backend failed: ${error.message}`);
}
}
async searchWithLiteBackend(query, options = {}) {
try {
const searchUrl = 'https://lite.duckduckgo.com/lite/';
const response = await axios_1.default.post(searchUrl, new URLSearchParams({
q: query,
kl: options.locale || 'en-us',
df: options.time || ''
}), {
headers: {
'Referer': 'https://lite.duckduckgo.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded'
},
timeout: 15000
});
const results = this.parseLiteResults(response.data);
return {
results: results.filter(r => !this.cache.has(r.href) && (this.cache.add(r.href), true)),
backend: Backend.LITE,
success: true,
timestamp: Date.now()
};
}
catch (error) {
throw new Error(`Lite backend failed: ${error.message}`);
}
}
parseHtmlResults(html) {
const results = [];
const resultPattern = /<div[^>]*class="[^"]*result[^"]*"[^>]*>([\s\S]*?)<\/div>/gi;
const linkPattern = /<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/i;
const snippetPattern = /<span[^>]*class="[^"]*snippet[^"]*"[^>]*>(.*?)<\/span>/i;
let match;
while ((match = resultPattern.exec(html)) !== null) {
const resultHtml = match[1];
const linkMatch = linkPattern.exec(resultHtml);
const snippetMatch = snippetPattern.exec(resultHtml);
if (linkMatch) {
const url = linkMatch[1];
const title = this.cleanText(linkMatch[2]);
const snippet = snippetMatch ? this.cleanText(snippetMatch[1]) : '';
if (url && title && !url.includes('google.com/search') && !url.includes('duckduckgo.com/y.js')) {
results.push({
title,
href: this.normalizeUrl(url),
body: this.cleanText(snippet),
});
}
}
}
return results;
}
parseLiteResults(html) {
const results = [];
const tablePattern = /<table[^>]*>([\s\S]*?)<\/table>/gi;
const rowPattern = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
let tableMatch;
while ((tableMatch = tablePattern.exec(html)) !== null) {
const tableHtml = tableMatch[1];
let rowMatch;
while ((rowMatch = rowPattern.exec(tableHtml)) !== null) {
const rowHtml = rowMatch[1];
const linkMatch = /<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/i.exec(rowHtml);
if (linkMatch) {
const url = linkMatch[1];
const title = this.cleanText(linkMatch[2]);
if (url && title && !url.includes('google.com/search') && !url.includes('duckduckgo.com/y.js')) {
results.push({
title,
href: this.normalizeUrl(url),
body: '',
});
}
}
}
}
return results;
}
cleanText(text) {
return text
.replace(/\s+/g, ' ')
.replace(/"/g, '"')
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/ /g, ' ')
.replace(/'/g, "'")
.trim();
}
async searchImages(query, options = {}) {
const imageQuery = `${query} images photos pictures`;
return this.search(imageQuery, options);
}
async searchNews(query, options = {}) {
const newsQuery = `${query} site:news.com OR site:bbc.com OR site:cnn.com OR site:reuters.com`;
return this.search(newsQuery, options);
}
async searchVideos(query, options = {}) {
const videoQuery = `${query} site:youtube.com OR site:vimeo.com OR site:dailymotion.com`;
return this.search(videoQuery, options);
}
}
exports.MultiBackendDuckDuckGoSearch = MultiBackendDuckDuckGoSearch;
exports.multiBackendSearch = new MultiBackendDuckDuckGoSearch();