n8n-nodes-crawl4ai
Version:
n8n nodes for Crawl4AI web crawler and data extraction
275 lines • 9.92 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.description = void 0;
exports.execute = execute;
const n8n_workflow_1 = require("n8n-workflow");
const utils_1 = require("../helpers/utils");
const formatters_1 = require("../helpers/formatters");
exports.description = [
{
displayName: 'URL',
name: 'url',
type: 'string',
required: true,
default: '',
placeholder: 'https://example.com',
description: 'The URL to crawl',
displayOptions: {
show: {
operation: ['crawlSingleUrl'],
},
},
},
{
displayName: 'Browser Options',
name: 'browserOptions',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['crawlSingleUrl'],
},
},
options: [
{
displayName: 'Enable JavaScript',
name: 'javaScriptEnabled',
type: 'boolean',
default: true,
description: 'Whether to enable JavaScript execution',
},
{
displayName: 'Headless Mode',
name: 'headless',
type: 'boolean',
default: true,
description: 'Whether to run browser in headless mode',
},
{
displayName: 'Timeout (Ms)',
name: 'timeout',
type: 'number',
default: 30000,
description: 'Maximum time to wait for the browser to load the page',
},
{
displayName: 'User Agent',
name: 'userAgent',
type: 'string',
default: '',
placeholder: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ...',
description: 'The user agent to use (leave empty for default)',
},
{
displayName: 'Viewport Height',
name: 'viewportHeight',
type: 'number',
default: 800,
description: 'The height of the browser viewport',
},
{
displayName: 'Viewport Width',
name: 'viewportWidth',
type: 'number',
default: 1280,
description: 'The width of the browser viewport',
},
],
},
{
displayName: 'Crawler Options',
name: 'crawlerOptions',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['crawlSingleUrl'],
},
},
options: [
{
displayName: 'Cache Mode',
name: 'cacheMode',
type: 'options',
options: [
{
name: 'Enabled (Read/Write)',
value: 'enabled',
description: 'Use cache if available, save new results to cache',
},
{
name: 'Bypass (Force Fresh)',
value: 'bypass',
description: 'Ignore cache, always fetch fresh content',
},
{
name: 'Only (Read Only)',
value: 'only',
description: 'Only use cache, do not make new requests',
},
],
default: 'enabled',
description: 'How to use the cache when crawling',
},
{
displayName: 'Check Robots.txt',
name: 'checkRobotsTxt',
type: 'boolean',
default: false,
description: 'Whether to respect robots.txt rules',
},
{
displayName: 'CSS Selector',
name: 'cssSelector',
type: 'string',
default: '',
placeholder: 'article.content',
description: 'CSS selector to focus on specific content (leave empty for full page)',
},
{
displayName: 'Exclude External Links',
name: 'excludeExternalLinks',
type: 'boolean',
default: false,
description: 'Whether to exclude external links from the result',
},
{
displayName: 'Excluded Tags',
name: 'excludedTags',
type: 'string',
default: '',
placeholder: 'nav,footer,aside',
description: 'Comma-separated list of HTML tags to exclude from processing',
},
{
displayName: 'JavaScript Code',
name: 'jsCode',
type: 'string',
typeOptions: {
rows: 4,
},
default: '',
placeholder: 'document.querySelector("button.load-more").click();',
description: 'JavaScript code to execute on the page after load',
},
{
displayName: 'JavaScript Only Mode',
name: 'jsOnly',
type: 'boolean',
default: false,
description: 'Whether to only execute JavaScript without crawling',
},
{
displayName: 'Max Retries',
name: 'maxRetries',
type: 'number',
default: 3,
description: 'Maximum number of retries for failed requests',
},
{
displayName: 'Page Timeout (Ms)',
name: 'pageTimeout',
type: 'number',
default: 30000,
description: 'Maximum time to wait for the page to load',
},
{
displayName: 'Request Timeout (Ms)',
name: 'requestTimeout',
type: 'number',
default: 30000,
description: 'Maximum time to wait for network requests',
},
{
displayName: 'Session ID',
name: 'sessionId',
type: 'string',
default: '',
placeholder: 'my-session-ID',
description: 'ID to maintain browser state across multiple crawls (for multi-step crawling)',
},
{
displayName: 'Word Count Threshold',
name: 'wordCountThreshold',
type: 'number',
default: 0,
description: 'Minimum number of words for content to be included',
},
],
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['crawlSingleUrl'],
},
},
options: [
{
displayName: 'Include Media Data',
name: 'includeMedia',
type: 'boolean',
default: false,
description: 'Whether to include media data in output (images, videos)',
},
{
displayName: 'Verbose Response',
name: 'verboseResponse',
type: 'boolean',
default: false,
description: 'Whether to include detailed data in output (HTML, status codes, etc.)',
},
],
},
];
async function execute(items, nodeOptions) {
var _a;
const allResults = [];
for (let i = 0; i < items.length; i++) {
try {
const url = this.getNodeParameter('url', i, '');
const browserOptions = this.getNodeParameter('browserOptions', i, {});
const crawlerOptions = this.getNodeParameter('crawlerOptions', i, {});
const options = this.getNodeParameter('options', i, {});
if (!url) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URL cannot be empty.', { itemIndex: i });
}
if (!(0, utils_1.isValidUrl)(url)) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URL: ${url}`, { itemIndex: i });
}
const browserConfig = (0, utils_1.createBrowserConfig)(browserOptions);
const crawlerConfig = (0, utils_1.createCrawlerRunConfig)({
...crawlerOptions,
...browserConfig,
});
const crawler = await (0, utils_1.getCrawl4aiClient)(this);
const result = await crawler.crawlUrl(url, crawlerConfig);
const formattedResult = (0, formatters_1.formatCrawlResult)(result, options.includeMedia, options.verboseResponse);
allResults.push({
json: formattedResult,
pairedItem: { item: i },
});
}
catch (error) {
if (this.continueOnFail()) {
const node = this.getNode();
const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
allResults.push({
json: items[i].json,
error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
pairedItem: { item: i },
});
continue;
}
throw error;
}
}
return allResults;
}
//# sourceMappingURL=crawlSingleUrl.operation.js.map