n8n-nodes-crawl4ai
Version:
n8n nodes for Crawl4AI web crawler and data extraction
90 lines • 3.54 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.isValidUrl = exports.safeJsonParse = exports.createCrawlerRunConfig = exports.createBrowserConfig = exports.getCrawl4aiClient = void 0;
exports.createCssSelectorExtractionStrategy = createCssSelectorExtractionStrategy;
exports.createLlmExtractionStrategy = createLlmExtractionStrategy;
exports.cleanExtractedData = cleanExtractedData;
var utils_1 = require("../../Crawl4aiBasicCrawler/helpers/utils");
Object.defineProperty(exports, "getCrawl4aiClient", { enumerable: true, get: function () { return utils_1.getCrawl4aiClient; } });
Object.defineProperty(exports, "createBrowserConfig", { enumerable: true, get: function () { return utils_1.createBrowserConfig; } });
Object.defineProperty(exports, "createCrawlerRunConfig", { enumerable: true, get: function () { return utils_1.createCrawlerRunConfig; } });
Object.defineProperty(exports, "safeJsonParse", { enumerable: true, get: function () { return utils_1.safeJsonParse; } });
Object.defineProperty(exports, "cleanText", { enumerable: true, get: function () { return utils_1.cleanText; } });
Object.defineProperty(exports, "isValidUrl", { enumerable: true, get: function () { return utils_1.isValidUrl; } });
function createCssSelectorExtractionStrategy(schema) {
return {
type: 'JsonCssExtractionStrategy',
params: {
schema: {
type: 'dict',
value: {
name: schema.name,
baseSelector: schema.baseSelector,
fields: schema.fields.map(field => ({
name: field.name,
selector: field.selector,
type: field.type,
attribute: field.attribute,
})),
},
},
},
};
}
function createLlmExtractionStrategy(schema, instruction, provider, apiKey) {
return {
type: 'LLMExtractionStrategy',
params: {
llm_config: {
type: 'LLMConfig',
params: {
provider: provider || 'openai/gpt-4o',
api_token: apiKey,
},
},
instruction,
schema: {
type: 'dict',
value: schema,
},
extraction_type: 'schema',
apply_chunking: false,
force_json_response: true,
},
};
}
function cleanExtractedData(data) {
if (!data)
return {};
const cleanedData = {};
Object.entries(data).forEach(([key, value]) => {
if (typeof value === 'string') {
cleanedData[key] = cleanText(value);
}
else if (Array.isArray(value)) {
cleanedData[key] = value.map(item => {
if (typeof item === 'string') {
return cleanText(item);
}
else if (typeof item === 'object' && item !== null) {
return cleanExtractedData(item);
}
return item;
});
}
else if (typeof value === 'object' && value !== null) {
cleanedData[key] = cleanExtractedData(value);
}
else {
cleanedData[key] = value;
}
});
return cleanedData;
}
function cleanText(value) {
return value
.replace(/\s+/g, ' ')
.replace(/^\s+|\s+$/g, '')
.trim();
}
//# sourceMappingURL=utils.js.map