n8n-nodes-crawl4ai
Version:
n8n nodes for Crawl4AI web crawler and data extraction
328 lines • 12.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.description = void 0;
exports.execute = execute;
const n8n_workflow_1 = require("n8n-workflow");
const utils_1 = require("../helpers/utils");
exports.description = [
{
displayName: 'URL',
name: 'url',
type: 'string',
required: true,
default: '',
placeholder: 'https://example.com/api/data.json',
description: 'The URL of the JSON data to extract',
displayOptions: {
show: {
operation: ['jsonExtractor'],
},
},
},
{
displayName: 'JSON Path',
name: 'jsonPath',
type: 'string',
default: '',
placeholder: 'data.items',
description: 'Path to the JSON data to extract (leave empty for entire JSON response)',
displayOptions: {
show: {
operation: ['jsonExtractor'],
},
},
},
{
displayName: 'Source Type',
name: 'sourceType',
type: 'options',
options: [
{
name: 'Direct JSON URL',
value: 'direct',
description: 'URL returns JSON directly',
},
{
name: 'JSON in Script Tag',
value: 'script',
description: 'JSON is embedded in a <script> tag',
},
{
name: 'JSON-LD',
value: 'jsonld',
description: 'JSON-LD structured data',
},
],
default: 'direct',
description: 'Where to find the JSON data on the page',
displayOptions: {
show: {
operation: ['jsonExtractor'],
},
},
},
{
displayName: 'Script Selector',
name: 'scriptSelector',
type: 'string',
default: '',
placeholder: 'script#__NEXT_DATA__',
description: 'CSS selector for the script tag containing JSON data',
displayOptions: {
show: {
operation: ['jsonExtractor'],
sourceType: ['script'],
},
},
},
{
displayName: 'Browser Options',
name: 'browserOptions',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['jsonExtractor'],
},
},
options: [
{
displayName: 'Headless Mode',
name: 'headless',
type: 'boolean',
default: true,
description: 'Whether to run browser in headless mode',
},
{
displayName: 'Enable JavaScript',
name: 'javaScriptEnabled',
type: 'boolean',
default: true,
description: 'Whether to enable JavaScript execution',
},
{
displayName: 'Timeout (MS)',
name: 'timeout',
type: 'number',
default: 30000,
description: 'Maximum time to wait for the browser to load the page',
},
{
displayName: 'JavaScript Code',
name: 'jsCode',
type: 'string',
typeOptions: {
rows: 4,
},
default: '',
placeholder: 'window.scrollTo(0, document.body.scrollHeight);',
description: 'JavaScript code to execute before extraction',
},
],
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
displayOptions: {
show: {
operation: ['jsonExtractor'],
},
},
options: [
{
displayName: 'Cache Mode',
name: 'cacheMode',
type: 'options',
options: [
{
name: 'Enabled (Read/Write)',
value: 'enabled',
description: 'Use cache if available, save new results to cache',
},
{
name: 'Bypass (Force Fresh)',
value: 'bypass',
description: 'Ignore cache, always fetch fresh content',
},
{
name: 'Only (Read Only)',
value: 'only',
description: 'Only use cache, do not make new requests',
},
],
default: 'enabled',
description: 'How to use the cache when crawling',
},
{
displayName: 'Include Full Content',
name: 'includeFullContent',
type: 'boolean',
default: false,
description: 'Whether to include the full JSON content in addition to the extracted data',
},
{
displayName: 'Headers',
name: 'headers',
type: 'string',
typeOptions: {
rows: 2,
},
default: '',
placeholder: '{"accept": "application/json"}',
description: 'Headers to send with the request (JSON format)',
},
],
},
];
async function execute(items, nodeOptions) {
var _a, _b;
const allResults = [];
for (let i = 0; i < items.length; i++) {
try {
const url = this.getNodeParameter('url', i, '');
const jsonPath = this.getNodeParameter('jsonPath', i, '');
const sourceType = this.getNodeParameter('sourceType', i, 'direct');
const scriptSelector = this.getNodeParameter('scriptSelector', i, '');
const browserOptions = this.getNodeParameter('browserOptions', i, {});
const options = this.getNodeParameter('options', i, {});
if (!url) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URL cannot be empty.', { itemIndex: i });
}
if (!(0, utils_1.isValidUrl)(url)) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URL: ${url}`, { itemIndex: i });
}
if (sourceType === 'script' && !scriptSelector) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Script selector is required when source type is "JSON in Script Tag".', { itemIndex: i });
}
let headers;
if (options.headers && typeof options.headers === 'string') {
try {
headers = JSON.parse(options.headers);
}
catch (error) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Headers must be a valid JSON object.', { itemIndex: i });
}
}
const browserConfig = (0, utils_1.createBrowserConfig)(browserOptions);
const crawler = await (0, utils_1.getCrawl4aiClient)(this);
const extractionStrategy = {
type: 'JsonCssExtractionStrategy',
params: {
schema: {
type: 'dict',
value: {
name: 'json_extraction',
baseSelector: sourceType === 'jsonld' ? 'script[type="application/ld+json"]' : scriptSelector || 'body',
fields: [
{
name: 'content',
selector: sourceType === 'jsonld' ? '' : scriptSelector || 'pre',
type: 'text',
}
],
},
},
},
};
const result = await crawler.arun(url, {
extractionStrategy: sourceType === 'direct' ? undefined : extractionStrategy,
browserConfig,
cacheMode: options.cacheMode || 'enabled',
jsCode: browserOptions.jsCode,
headers,
});
let jsonData = null;
if (result.success) {
if (sourceType === 'direct') {
try {
if (result.extracted_content) {
jsonData = JSON.parse(result.extracted_content);
}
else if (result.text) {
jsonData = JSON.parse(result.text);
}
}
catch (error) {
const jsonMatch = (_a = result.text) === null || _a === void 0 ? void 0 : _a.match(/\{[\s\S]*\}|\[[\s\S]*\]/);
if (jsonMatch) {
try {
jsonData = JSON.parse(jsonMatch[0]);
}
catch {
jsonData = { content: result.text };
}
}
}
}
else {
if (result.extracted_content) {
try {
const extractedData = JSON.parse(result.extracted_content);
if (Array.isArray(extractedData) && extractedData.length > 0) {
const content = extractedData[0].content;
jsonData = JSON.parse(content);
}
}
catch (error) {
jsonData = { error: 'Failed to parse JSON from script tag' };
}
}
}
if (jsonPath && jsonData) {
const pathParts = jsonPath.split('.');
let currentData = jsonData;
for (const part of pathParts) {
if (currentData && typeof currentData === 'object' && part in currentData) {
currentData = currentData[part];
}
else {
currentData = null;
break;
}
}
jsonData = currentData;
}
}
const output = {
url,
success: result.success,
};
if (!result.success && result.error_message) {
output.error = result.error_message;
}
else if (jsonData) {
output.data = jsonData;
if (options.includeFullContent === true) {
output.fullContent = result.text || result.extracted_content;
}
}
else {
output.error = 'No JSON data found or failed to parse JSON';
output.success = false;
}
allResults.push({
json: output,
pairedItem: { item: i },
});
}
catch (error) {
if (this.continueOnFail()) {
const node = this.getNode();
const errorItemIndex = (_b = error.itemIndex) !== null && _b !== void 0 ? _b : i;
allResults.push({
json: items[i].json,
error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
pairedItem: { item: i },
});
continue;
}
throw error;
}
}
return allResults;
}
//# sourceMappingURL=jsonExtractor.operation.js.map