n8n-nodes-duckduckgo-search
Version:
A powerful and comprehensive n8n community node that seamlessly integrates DuckDuckGo search capabilities into your workflows. Search the web, find images, discover news, and explore videos - all with privacy-focused, reliable results.
269 lines (268 loc) • 12.1 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.cleanupExpiredVqdTokens = exports.getVqdCacheStats = exports.getVqdTokenCacheSize = exports.clearVqdTokenCache = exports.paginateWithVqd = exports.DEFAULT_PAGINATION_CONFIG = void 0;
const duck_duck_scrape_1 = require("duck-duck-scrape");
const utils_1 = require("./utils");
const errors_1 = require("./errors");
exports.DEFAULT_PAGINATION_CONFIG = {
pageSize: 10,
maxPages: 15,
delayBetweenRequests: 750,
maxConsecutiveErrors: 3,
vqdTokenMaxUsage: 25,
vqdTokenLifetime: 4 * 60 * 1000,
};
const vqdTokenCache = new Map();
function getCachedVqdToken(query) {
const cached = vqdTokenCache.get(query);
if (cached) {
const tokenAge = Date.now() - cached.lastUpdated;
const isExpired = tokenAge > exports.DEFAULT_PAGINATION_CONFIG.vqdTokenLifetime;
const isOverused = cached.usageCount >= cached.maxUsage;
if (isExpired || isOverused) {
vqdTokenCache.delete(query);
return undefined;
}
return cached;
}
return undefined;
}
function updateVqdTokenCache(query, token, incrementUsage = false) {
if (token) {
const existing = vqdTokenCache.get(query);
if (existing && incrementUsage) {
existing.usageCount++;
existing.lastUpdated = Date.now();
}
else {
vqdTokenCache.set(query, {
token,
lastUpdated: Date.now(),
query,
usageCount: 1,
maxUsage: exports.DEFAULT_PAGINATION_CONFIG.vqdTokenMaxUsage,
});
}
}
}
async function sleepWithJitter(baseDelay) {
const jitter = Math.random() * 0.5 * baseDelay;
const totalDelay = baseDelay + jitter;
return new Promise(resolve => setTimeout(resolve, totalDelay));
}
async function paginateWithVqd(query, baseOptions, paginationOptions) {
var _a, _b, _c, _d;
const results = [];
const errors = [];
let strategy = 'primary';
let vqdManager = getCachedVqdToken(query);
let currentPage = 0;
let hasMore = true;
let consecutiveErrors = 0;
const { maxResults, pageSize, maxPages, delayBetweenRequests, debugMode, useBackupStrategy = true } = paginationOptions;
try {
if (!(vqdManager === null || vqdManager === void 0 ? void 0 : vqdManager.token)) {
const initialResult = await getInitialResults(query, baseOptions, debugMode);
if (initialResult.success && initialResult.data) {
results.push(...initialResult.data.results);
if (initialResult.data.vqd) {
updateVqdTokenCache(query, initialResult.data.vqd);
vqdManager = getCachedVqdToken(query);
}
currentPage++;
consecutiveErrors = 0;
}
else {
errors.push(initialResult.error || 'Failed to get initial results');
if (!useBackupStrategy) {
hasMore = false;
}
}
await sleepWithJitter(delayBetweenRequests);
}
while (results.length < maxResults &&
hasMore &&
currentPage < maxPages &&
consecutiveErrors < exports.DEFAULT_PAGINATION_CONFIG.maxConsecutiveErrors) {
let pageResult = null;
let pageSuccess = false;
if (vqdManager === null || vqdManager === void 0 ? void 0 : vqdManager.token) {
try {
pageResult = await getPageWithVqd(query, baseOptions, vqdManager, currentPage, pageSize, debugMode);
if (((_a = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _a === void 0 ? void 0 : _a.length) > 0) {
results.push(...pageResult.results);
pageSuccess = true;
consecutiveErrors = 0;
if (pageResult.vqd && pageResult.vqd !== vqdManager.token) {
updateVqdTokenCache(query, pageResult.vqd);
vqdManager = getCachedVqdToken(query);
}
else if (vqdManager.token) {
updateVqdTokenCache(query, vqdManager.token, true);
vqdManager = getCachedVqdToken(query);
}
}
}
catch (vqdError) {
errors.push(`VQD pagination error on page ${currentPage + 1}: ${vqdError.message}`);
if (debugMode) {
console.error(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.ERROR, `VQD pagination failed: ${vqdError.message}`, 'paginateWithVqd', { query, page: currentPage + 1, error: vqdError.message })));
}
if (vqdError.message.includes('VQD') || vqdError.message.includes('token')) {
vqdTokenCache.delete(query);
vqdManager = undefined;
}
}
}
if (!pageSuccess && useBackupStrategy) {
try {
strategy = vqdManager ? 'hybrid' : 'fallback';
pageResult = await getFallbackResults(query, baseOptions, currentPage, pageSize, debugMode);
if (((_b = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _b === void 0 ? void 0 : _b.length) > 0) {
results.push(...pageResult.results);
pageSuccess = true;
consecutiveErrors = 0;
}
}
catch (fallbackError) {
errors.push(`Fallback strategy failed on page ${currentPage + 1}: ${fallbackError.message}`);
if (debugMode) {
console.error(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.ERROR, `Fallback pagination failed: ${fallbackError.message}`, 'paginateWithVqd', { query, page: currentPage + 1, error: fallbackError.message })));
}
}
}
if (!pageSuccess) {
consecutiveErrors++;
if (consecutiveErrors >= exports.DEFAULT_PAGINATION_CONFIG.maxConsecutiveErrors) {
hasMore = false;
errors.push(`Maximum consecutive errors reached (${consecutiveErrors})`);
break;
}
}
else {
const expectedMinResults = Math.min(pageSize * 0.5, 3);
if (((_c = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _c === void 0 ? void 0 : _c.length) < expectedMinResults) {
hasMore = false;
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Received fewer results than expected, assuming end of results`, 'paginateWithVqd', {
query,
page: currentPage + 1,
received: (_d = pageResult === null || pageResult === void 0 ? void 0 : pageResult.results) === null || _d === void 0 ? void 0 : _d.length,
expected: pageSize
})));
}
}
}
currentPage++;
if (hasMore && currentPage < maxPages) {
await sleepWithJitter(delayBetweenRequests);
}
}
}
catch (error) {
errors.push(`Pagination error: ${error.message}`);
throw new errors_1.DuckDuckGoError(`Pagination failed: ${error.message}`, errors_1.DuckDuckGoErrorType.PAGINATION_ERROR, {
technicalDetails: { query, currentPage, errors },
});
}
return {
results: results.slice(0, maxResults),
totalFetched: results.length,
pagesProcessed: currentPage,
vqdToken: vqdManager === null || vqdManager === void 0 ? void 0 : vqdManager.token,
hasMore: hasMore && results.length >= maxResults,
strategy,
errors,
};
}
exports.paginateWithVqd = paginateWithVqd;
async function getInitialResults(query, baseOptions, debugMode) {
var _a;
try {
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Fetching initial results for: ${query}`, 'getInitialResults', { query, options: baseOptions })));
}
const result = await (0, duck_duck_scrape_1.search)(query, baseOptions);
if (((_a = result === null || result === void 0 ? void 0 : result.results) === null || _a === void 0 ? void 0 : _a.length) > 0) {
return { success: true, data: result };
}
else {
return { success: false, error: 'No initial results found' };
}
}
catch (error) {
return { success: false, error: error.message };
}
}
async function getPageWithVqd(query, baseOptions, vqdManager, currentPage, pageSize, debugMode) {
const offset = currentPage * pageSize;
const paginationOptions = {
...baseOptions,
vqd: vqdManager.token,
offset: offset,
};
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Fetching page ${currentPage + 1} with VQD token`, 'getPageWithVqd', {
query,
page: currentPage + 1,
offset,
vqdUsage: vqdManager.usageCount,
vqdToken: vqdManager.token ? vqdManager.token.substring(0, 10) + '...' : 'undefined'
})));
}
return await (0, duck_duck_scrape_1.search)(query, paginationOptions);
}
async function getFallbackResults(query, baseOptions, currentPage, pageSize, debugMode) {
if (debugMode) {
console.log(JSON.stringify((0, utils_1.createLogEntry)(utils_1.LogLevel.INFO, `Using fallback strategy for page ${currentPage + 1}`, 'getFallbackResults', { query, page: currentPage + 1 })));
}
const basicOptions = { ...baseOptions };
delete basicOptions.vqd;
delete basicOptions.offset;
return await (0, duck_duck_scrape_1.search)(query, basicOptions);
}
function clearVqdTokenCache() {
vqdTokenCache.clear();
}
exports.clearVqdTokenCache = clearVqdTokenCache;
function getVqdTokenCacheSize() {
return vqdTokenCache.size;
}
exports.getVqdTokenCacheSize = getVqdTokenCacheSize;
function getVqdCacheStats() {
const now = Date.now();
let activeTokens = 0;
let expiredTokens = 0;
for (const [, manager] of vqdTokenCache.entries()) {
const age = now - manager.lastUpdated;
const isExpired = age > exports.DEFAULT_PAGINATION_CONFIG.vqdTokenLifetime ||
manager.usageCount >= manager.maxUsage;
if (isExpired) {
expiredTokens++;
}
else {
activeTokens++;
}
}
return {
totalTokens: vqdTokenCache.size,
activeTokens,
expiredTokens,
};
}
exports.getVqdCacheStats = getVqdCacheStats;
function cleanupExpiredVqdTokens() {
const now = Date.now();
let removedCount = 0;
for (const [query, manager] of vqdTokenCache.entries()) {
const age = now - manager.lastUpdated;
const isExpired = age > exports.DEFAULT_PAGINATION_CONFIG.vqdTokenLifetime ||
manager.usageCount >= manager.maxUsage;
if (isExpired) {
vqdTokenCache.delete(query);
removedCount++;
}
}
return removedCount;
}
exports.cleanupExpiredVqdTokens = cleanupExpiredVqdTokens;