n8n-nodes-crawl4ai
Version:
n8n nodes for Crawl4AI web crawler and data extraction
80 lines • 2.95 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.getCrawl4aiClient = getCrawl4aiClient;
exports.createBrowserConfig = createBrowserConfig;
exports.createCrawlerRunConfig = createCrawlerRunConfig;
exports.safeJsonParse = safeJsonParse;
exports.cleanText = cleanText;
exports.isValidUrl = isValidUrl;
const apiClient_1 = require("./apiClient");
async function getCrawl4aiClient(executeFunctions) {
const credentials = await executeFunctions.getCredentials('crawl4aiApi');
if (!credentials) {
throw new Error('Crawl4AI credentials are not configured!');
}
return (0, apiClient_1.createCrawlerInstance)(credentials);
}
function createBrowserConfig(options) {
return {
headless: options.headless !== false,
javaScriptEnabled: options.javaScriptEnabled === true,
viewport: {
width: options.viewportWidth ? Number(options.viewportWidth) : 1280,
height: options.viewportHeight ? Number(options.viewportHeight) : 800,
},
timeout: options.timeout ? Number(options.timeout) : 30000,
userAgent: options.userAgent ? String(options.userAgent) : undefined,
};
}
function createCrawlerRunConfig(options) {
let excludedTags = [];
if (options.excludedTags) {
if (typeof options.excludedTags === 'string') {
excludedTags = options.excludedTags
.split(',')
.map(tag => tag.trim())
.filter(tag => tag);
}
else if (Array.isArray(options.excludedTags)) {
excludedTags = options.excludedTags;
}
}
return {
cacheMode: options.cacheMode || 'enabled',
streamEnabled: options.streamEnabled === true,
pageTimeout: options.pageTimeout ? Number(options.pageTimeout) : 30000,
requestTimeout: options.requestTimeout ? Number(options.requestTimeout) : 30000,
jsCode: options.jsCode ? String(options.jsCode) : undefined,
jsOnly: options.jsOnly === true,
cssSelector: options.cssSelector ? String(options.cssSelector) : undefined,
excludedTags,
excludeExternalLinks: options.excludeExternalLinks === true,
checkRobotsTxt: options.checkRobotsTxt === true,
wordCountThreshold: options.wordCountThreshold ? Number(options.wordCountThreshold) : 0,
sessionId: options.sessionId ? String(options.sessionId) : undefined,
maxRetries: options.maxRetries ? Number(options.maxRetries) : 3,
};
}
function safeJsonParse(jsonString, defaultValue = null) {
try {
return JSON.parse(jsonString);
}
catch (error) {
return defaultValue;
}
}
function cleanText(text) {
return text
.replace(/\s+/g, ' ')
.trim();
}
function isValidUrl(url) {
try {
new URL(url);
return true;
}
catch (error) {
return false;
}
}
//# sourceMappingURL=utils.js.map