n8n-nodes-duckduckgo-search
Version:
AI Agent-ready n8n community node for DuckDuckGo search. Search the web, images, news, and videos with no API key required and no outbound telemetry. Optionally fetch and extract the main text of result pages or any URL, and get DuckDuckGo Instant Answers
269 lines (268 loc) • 11.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.DEFAULT_PAGINATION_CONFIG = void 0;
exports.paginateWithVqd = paginateWithVqd;
exports.clearVqdTokenCache = clearVqdTokenCache;
exports.getVqdTokenCacheSize = getVqdTokenCacheSize;
exports.getVqdCacheStats = getVqdCacheStats;
exports.cleanupExpiredVqdTokens = cleanupExpiredVqdTokens;
const duck_duck_scrape_1 = require("duck-duck-scrape");
const utils_1 = require("./utils");
const errors_1 = require("./errors");
exports.DEFAULT_PAGINATION_CONFIG = {
pageSize: 10,
maxPages: 15,
delayBetweenRequests: 750,
maxConsecutiveErrors: 3,
vqdTokenMaxUsage: 25,
vqdTokenLifetime: 4 * 60 * 1000,
};
const vqdTokenCache = new Map();
function getCachedVqdToken(query) {
const cached = vqdTokenCache.get(query);
if (cached) {
const tokenAge = Date.now() - cached.lastUpdated;
const isExpired = tokenAge > exports.DEFAULT_PAGINATION_CONFIG.vqdTokenLifetime;
const isOverused = cached.usageCount >= cached.maxUsage;
if (isExpired || isOverused) {
vqdTokenCache.delete(query);
return undefined;
}
return cached;
}
return undefined;
}
function updateVqdTokenCache(query, token, incrementUsage = false) {
if (token) {
const existing = vqdTokenCache.get(query);
if (existing && incrementUsage) {
existing.usageCount++;
existing.lastUpdated = Date.now();
}
else {
vqdTokenCache.set(query, {
token,
lastUpdated: Date.now(),
query,
usageCount: 1,
maxUsage: exports.DEFAULT_PAGINATION_CONFIG.vqdTokenMaxUsage,
});
}
}
}
async function sleepWithJitter(baseDelay) {
const jitter = Math.random() * 0.5 * baseDelay;
const totalDelay = baseDelay + jitter;
return new Promise(resolve => setTimeout(resolve, totalDelay));
}
async function paginateWithVqd(query, baseOptions, paginationOptions) {
var _a, _b, _c, _d;
const results = [];
const errors = [];
let strategy = 'primary';
let vqdManager = getCachedVqdToken(query);
let currentPage = 0;
let hasMore = true;
let consecutiveErrors = 0;
const { maxResults, pageSize, maxPages, delayBetweenRequests, debugMode, useBackupStrategy = true } = paginationOptions;
try {
if (!(vqdManager === null || vqdManager === void 0 ? void 0 : vqdManager.token)) {
const initialResult = await getInitialResults(query, baseOptions, debugMode);
if (initialResult.success && initialResult.data) {
results.push(...initialResult.data.results);
if (initialResult.data.vqd) {
updateVqdTokenCache(query, initialResult.data.vqd);
vqdManager = getCachedVqdToken(query);
}
currentPage++;
consecutiveErrors = 0;
}
else {
errors.push(initialResult.error || 'Failed to get initial results');
if (!useBackupStrategy) {
hasMore = false;
}
}
await sleepWithJitter(delayBetweenRequests);
}
while (results.length < maxResults &&
hasMore &&
currentPage < maxPages &&
consecutiveErrors < exports.DEFAULT_PAGINATION_CONFIG.maxConsecutiveErrors) {
let pageResult = null;
let pageSuccess = false;
if (vqdManager === null || vqdManager === void 0 ? void 0 : vqdManager.token) {
try {
pageResult = await getPageWithVqd(query, baseOptions, vqdManager, currentPage, pageSize, debugMode);
if (((_a = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _a === void 0 ? void 0 : _a.length) > 0) {
results.push(...pageResult.results);
pageSuccess = true;
consecutiveErrors = 0;
if (pageResult.vqd && pageResult.vqd !== vqdManager.token) {
updateVqdTokenCache(query, pageResult.vqd);
vqdManager = getCachedVqdToken(query);
}
else if (vqdManager.token) {
updateVqdTokenCache(query, vqdManager.token, true);
vqdManager = getCachedVqdToken(query);
}
}
}
catch (vqdError) {
errors.push(`VQD pagination error on page ${currentPage + 1}: ${vqdError.message}`);
if (debugMode) {
console.error(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.ERROR, `VQD pagination failed: ${vqdError.message}`, 'paginateWithVqd', { query, page: currentPage + 1, error: vqdError.message })));
}
if (vqdError.message.includes('VQD') || vqdError.message.includes('token')) {
vqdTokenCache.delete(query);
vqdManager = undefined;
}
}
}
if (!pageSuccess && useBackupStrategy) {
try {
strategy = vqdManager ? 'hybrid' : 'fallback';
pageResult = await getFallbackResults(query, baseOptions, currentPage, pageSize, debugMode);
if (((_b = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _b === void 0 ? void 0 : _b.length) > 0) {
results.push(...pageResult.results);
pageSuccess = true;
consecutiveErrors = 0;
}
}
catch (fallbackError) {
errors.push(`Fallback strategy failed on page ${currentPage + 1}: ${fallbackError.message}`);
if (debugMode) {
console.error(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.ERROR, `Fallback pagination failed: ${fallbackError.message}`, 'paginateWithVqd', { query, page: currentPage + 1, error: fallbackError.message })));
}
}
}
if (!pageSuccess) {
consecutiveErrors++;
if (consecutiveErrors >= exports.DEFAULT_PAGINATION_CONFIG.maxConsecutiveErrors) {
hasMore = false;
errors.push(`Maximum consecutive errors reached (${consecutiveErrors})`);
break;
}
}
else {
const expectedMinResults = Math.min(pageSize * 0.5, 3);
if (((_c = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _c === void 0 ? void 0 : _c.length) < expectedMinResults) {
hasMore = false;
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Received fewer results than expected, assuming end of results`, 'paginateWithVqd', {
query,
page: currentPage + 1,
received: (_d = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _d === void 0 ? void 0 : _d.length,
expected: pageSize
})));
}
}
}
currentPage++;
if (hasMore && currentPage < maxPages) {
await sleepWithJitter(delayBetweenRequests);
}
}
}
catch (error) {
errors.push(`Pagination error: ${error.message}`);
throw new errors_1.DuckDuckGoError(`Pagination failed: ${error.message}`, errors_1.DuckDuckGoErrorType.PAGINATION_ERROR, {
technicalDetails: { query, currentPage, errors },
});
}
return {
results: results.slice(0, maxResults),
totalFetched: results.length,
pagesProcessed: currentPage,
vqdToken: vqdManager === null || vqdManager === void 0 ? void 0 : vqdManager.token,
hasMore: hasMore && results.length >= maxResults,
strategy,
errors,
};
}
async function getInitialResults(query, baseOptions, debugMode) {
var _a;
try {
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Fetching initial results for: ${query}`, 'getInitialResults', { query, options: baseOptions })));
}
const result = await (0, duck_duck_scrape_1.search)(query, baseOptions);
if (((_a = result === null || result === void 0 ? void 0 : result.results) === null || _a === void 0 ? void 0 : _a.length) > 0) {
return { success: true, data: result };
}
else {
return { success: false, error: 'No initial results found' };
}
}
catch (error) {
return { success: false, error: error.message };
}
}
async function getPageWithVqd(query, baseOptions, vqdManager, currentPage, pageSize, debugMode) {
const offset = currentPage * pageSize;
const paginationOptions = {
...baseOptions,
vqd: vqdManager.token,
offset: offset,
};
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Fetching page ${currentPage + 1} with VQD token`, 'getPageWithVqd', {
query,
page: currentPage + 1,
offset,
vqdUsage: vqdManager.usageCount,
vqdToken: vqdManager.token ? vqdManager.token.substring(0, 10) + '...' : 'undefined'
})));
}
return await (0, duck_duck_scrape_1.search)(query, paginationOptions);
}
async function getFallbackResults(query, baseOptions, currentPage, pageSize, debugMode) {
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Using fallback strategy for page ${currentPage + 1}`, 'getFallbackResults', { query, page: currentPage + 1 })));
}
const basicOptions = { ...baseOptions };
delete basicOptions.vqd;
delete basicOptions.offset;
return await (0, duck_duck_scrape_1.search)(query, basicOptions);
}
function clearVqdTokenCache() {
vqdTokenCache.clear();
}
function getVqdTokenCacheSize() {
return vqdTokenCache.size;
}
function getVqdCacheStats() {
const now = Date.now();
let activeTokens = 0;
let expiredTokens = 0;
for (const [, manager] of vqdTokenCache.entries()) {
const age = now - manager.lastUpdated;
const isExpired = age > exports.DEFAULT_PAGINATION_CONFIG.vqdTokenLifetime ||
manager.usageCount >= manager.maxUsage;
if (isExpired) {
expiredTokens++;
}
else {
activeTokens++;
}
}
return {
totalTokens: vqdTokenCache.size,
activeTokens,
expiredTokens,
};
}
function cleanupExpiredVqdTokens() {
const now = Date.now();
let removedCount = 0;
for (const [query, manager] of vqdTokenCache.entries()) {
const age = now - manager.lastUpdated;
const isExpired = age > exports.DEFAULT_PAGINATION_CONFIG.vqdTokenLifetime ||
manager.usageCount >= manager.maxUsage;
if (isExpired) {
vqdTokenCache.delete(query);
removedCount++;
}
}
return removedCount;
}