n8n-nodes-crawl4ai
Version:
n8n nodes for Crawl4AI web crawler and data extraction
148 lines • 5.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.description = void 0;
exports.execute = execute;
const n8n_workflow_1 = require("n8n-workflow");
const utils_1 = require("../helpers/utils");
const formatters_1 = require("../helpers/formatters");
exports.description = [
{
displayName: 'HTML Content',
name: 'html',
type: 'string',
typeOptions: {
rows: 8,
},
required: true,
default: '',
placeholder: '<html><body><h1>Example</h1><p>Content</p></body></html>',
description: 'The raw HTML content to process',
displayOptions: {
show: {
operation: ['processRawHtml'],
},
},
},
{
displayName: 'Base URL',
name: 'baseUrl',
type: 'string',
default: 'https://example.com',
description: 'The base URL to use for resolving relative links',
displayOptions: {
show: {
operation: ['processRawHtml'],
},
},
},
{
displayName: 'Crawler Options',
name: 'crawlerOptions',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['processRawHtml'],
},
},
options: [
{
displayName: 'CSS Selector',
name: 'cssSelector',
type: 'string',
default: '',
placeholder: 'article.content',
description: 'CSS selector to focus on specific content (leave empty for full page)',
},
{
displayName: 'Exclude External Links',
name: 'excludeExternalLinks',
type: 'boolean',
default: false,
description: 'Whether to exclude external links from the result',
},
{
displayName: 'Excluded Tags',
name: 'excludedTags',
type: 'string',
default: '',
placeholder: 'nav,footer,aside',
description: 'Comma-separated list of HTML tags to exclude from processing',
},
{
displayName: 'Word Count Threshold',
name: 'wordCountThreshold',
type: 'number',
default: 0,
description: 'Minimum number of words for content to be included',
},
],
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['processRawHtml'],
},
},
options: [
{
displayName: 'Include Media Data',
name: 'includeMedia',
type: 'boolean',
default: false,
description: 'Whether to include media data in output (images, videos)',
},
{
displayName: 'Verbose Response',
name: 'verboseResponse',
type: 'boolean',
default: false,
description: 'Whether to include detailed data in output (HTML, status codes, etc.)',
},
],
},
];
async function execute(items, nodeOptions) {
var _a;
const allResults = [];
for (let i = 0; i < items.length; i++) {
try {
const html = this.getNodeParameter('html', i, '');
const baseUrl = this.getNodeParameter('baseUrl', i, 'https://example.com');
const crawlerOptions = this.getNodeParameter('crawlerOptions', i, {});
const options = this.getNodeParameter('options', i, {});
if (!html) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'HTML content cannot be empty.', { itemIndex: i });
}
const crawlerConfig = (0, utils_1.createCrawlerRunConfig)(crawlerOptions);
const crawler = await (0, utils_1.getCrawl4aiClient)(this);
const result = await crawler.processRawHtml(html, baseUrl, crawlerConfig);
const formattedResult = (0, formatters_1.formatCrawlResult)(result, options.includeMedia, options.verboseResponse);
allResults.push({
json: formattedResult,
pairedItem: { item: i },
});
}
catch (error) {
if (this.continueOnFail()) {
const node = this.getNode();
const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
allResults.push({
json: items[i].json,
error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
pairedItem: { item: i },
});
continue;
}
throw error;
}
}
return allResults;
}
//# sourceMappingURL=processRawHtml.operation.js.map