n8n-nodes-duckduckgo-search
Version:
AI Agent-ready n8n community node for DuckDuckGo search. Search the web, images, news, and videos with no API key required and no outbound telemetry. Optionally fetch and extract the main text of result pages or any URL, and get DuckDuckGo Instant Answers
207 lines (206 loc) • 8.47 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.decodeEntities = decodeEntities;
exports.extractMainText = extractMainText;
exports.extractWithDomHeuristic = extractWithDomHeuristic;
exports.extractWithReadability = extractWithReadability;
exports.truncateText = truncateText;
exports.fetchPageContent = fetchPageContent;
exports.fetchPageContents = fetchPageContents;
const axios_1 = __importDefault(require("axios"));
const readability_1 = require("@mozilla/readability");
const linkedom_1 = require("linkedom");
const constants_1 = require("./constants");
const DEFAULTS = {
timeout: 8000,
maxLength: 2000,
maxBytes: 2 * 1024 * 1024,
};
const NAMED_ENTITIES = {
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
ndash: '–', mdash: '—', hellip: '…', laquo: '«', raquo: '»',
lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”',
copy: '©', reg: '®', trade: '™', deg: '°', euro: '€', pound: '£', cent: '¢',
};
function decodeEntities(text) {
return text.replace(/&(#[xX]?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (match, body) => {
if (body[0] === '#') {
const isHex = body[1] === 'x' || body[1] === 'X';
const code = parseInt(body.slice(isHex ? 2 : 1), isHex ? 16 : 10);
if (Number.isNaN(code))
return match;
try {
return String.fromCodePoint(code);
}
catch {
return match;
}
}
const named = NAMED_ENTITIES[body.toLowerCase()];
return named !== undefined ? named : match;
});
}
function extractMainText(html) {
if (!html)
return '';
let s = html;
s = s.replace(/<!--[\s\S]*?-->/g, ' ');
s = s.replace(/<(script|style|noscript|template|svg|head|nav|footer|header|aside|form|iframe|button|select|figure)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ');
const body = s.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
if (body)
s = body[1];
return htmlFragmentToText(s);
}
function htmlFragmentToText(html) {
let s = html;
s = s.replace(/<\/(p|div|section|article|h[1-6]|li|tr|blockquote|td|th|pre)>/gi, '\n');
s = s.replace(/<br\s*\/?>/gi, '\n');
s = s.replace(/<[^>]+>/g, ' ');
s = decodeEntities(s);
return normaliseWhitespace(s);
}
const BOILERPLATE_SELECTOR = 'script,style,noscript,template,svg,head,nav,footer,header,aside,form,iframe,button,select,figure';
const LINK_DENSITY_SELECTOR = 'ul,ol,div,section,table';
function extractWithDomHeuristic(html) {
if (!html)
return null;
try {
const doc = (0, linkedom_1.parseHTML)(html).document;
Array.from(doc.querySelectorAll(BOILERPLATE_SELECTOR)).forEach((el) => el.remove());
Array.from(doc.querySelectorAll(LINK_DENSITY_SELECTOR)).forEach((el) => {
const text = (el.textContent || '').replace(/\s+/g, ' ').trim();
if (!text || text.length > 2000)
return;
const linkText = Array.from(el.querySelectorAll('a'))
.map((a) => a.textContent || '')
.join('')
.replace(/\s+/g, ' ')
.trim();
if (linkText.length / text.length > 0.5)
el.remove();
});
const container = doc.querySelector('article') || doc.querySelector('main') || doc.body;
const fragment = container ? container.innerHTML : '';
const text = htmlFragmentToText(fragment);
return text.length > 0 ? text : null;
}
catch {
return null;
}
}
function normaliseWhitespace(s) {
return s
.replace(/\r\n?/g, '\n')
.replace(/[^\S\n]+/g, ' ')
.replace(/ *\n */g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim();
}
const MIN_READABLE_LENGTH = 200;
function extractWithReadability(html) {
var _a;
if (!html)
return null;
try {
const { document } = (0, linkedom_1.parseHTML)(html);
const article = new readability_1.Readability(document).parse();
if (article && article.textContent && ((_a = article.length) !== null && _a !== void 0 ? _a : 0) >= MIN_READABLE_LENGTH) {
const text = normaliseWhitespace(article.textContent);
if (text.length === 0)
return null;
const meta = {};
if (article.title)
meta.title = article.title;
if (article.byline)
meta.author = article.byline;
if (article.excerpt)
meta.excerpt = article.excerpt;
if (article.publishedTime)
meta.published = article.publishedTime;
if (article.siteName)
meta.siteName = article.siteName;
return { text, meta };
}
return null;
}
catch {
return null;
}
}
function truncateText(text, maxLength) {
if (!maxLength || maxLength <= 0 || text.length <= maxLength) {
return { text, truncated: false };
}
let cut = text.slice(0, maxLength);
const lastSpace = cut.lastIndexOf(' ');
if (lastSpace > maxLength * 0.6) {
cut = cut.slice(0, lastSpace);
}
return { text: `${cut.trimEnd()}…`, truncated: true };
}
function isHtmlContentType(contentType) {
if (typeof contentType !== 'string' || contentType === '')
return true;
const ct = contentType.toLowerCase();
return ct.includes('text/html') || ct.includes('application/xhtml') || ct.includes('text/plain');
}
async function fetchPageContent(url, options = {}) {
var _a, _b, _c, _d, _e, _f, _g;
const timeout = (_a = options.timeout) !== null && _a !== void 0 ? _a : DEFAULTS.timeout;
const maxLength = (_b = options.maxLength) !== null && _b !== void 0 ? _b : DEFAULTS.maxLength;
const maxBytes = (_c = options.maxBytes) !== null && _c !== void 0 ? _c : DEFAULTS.maxBytes;
if (!url || typeof url !== 'string') {
return { content: '', truncated: false, error: 'No URL to fetch' };
}
try {
const response = await axios_1.default.get(url, {
timeout,
responseType: 'text',
maxContentLength: maxBytes,
maxBodyLength: maxBytes,
validateStatus: (status) => status >= 200 && status < 300,
headers: {
'User-Agent': constants_1.BROWSER_USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
},
});
const contentType = (_d = response.headers) === null || _d === void 0 ? void 0 : _d['content-type'];
if (!isHtmlContentType(contentType)) {
return { content: '', truncated: false, error: `Unsupported content type: ${contentType}` };
}
const html = typeof response.data === 'string' ? response.data : String((_e = response.data) !== null && _e !== void 0 ? _e : '');
const readable = extractWithReadability(html);
const rawText = readable
? readable.text
: ((_f = extractWithDomHeuristic(html)) !== null && _f !== void 0 ? _f : extractMainText(html));
const { text: finalText, truncated } = truncateText(rawText, maxLength);
const result = { content: finalText, truncated };
if (readable && Object.keys(readable.meta).length > 0) {
result.meta = readable.meta;
}
return result;
}
catch (error) {
let message;
if ((error === null || error === void 0 ? void 0 : error.code) === 'ECONNABORTED') {
message = `Timed out after ${timeout}ms`;
}
else if ((_g = error === null || error === void 0 ? void 0 : error.response) === null || _g === void 0 ? void 0 : _g.status) {
message = `HTTP ${error.response.status}`;
}
else if (error === null || error === void 0 ? void 0 : error.code) {
message = String(error.code);
}
else {
message = error instanceof Error ? error.message : 'Unknown error';
}
return { content: '', truncated: false, error: message };
}
}
async function fetchPageContents(urls, options = {}) {
return Promise.all(urls.map((u) => fetchPageContent(u, options)));
}