n8n-nodes-crawl4ai
Version:
n8n nodes for Crawl4AI web crawler and data extraction
223 lines • 8.88 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Crawl4aiClient = void 0;
exports.createCrawlerInstance = createCrawlerInstance;
const axios_1 = __importDefault(require("axios"));
class Crawl4aiClient {
constructor(credentials) {
this.credentials = credentials;
this.apiClient = this.createApiClient();
}
createApiClient() {
const baseURL = this.credentials.connectionMode === 'docker'
? this.credentials.dockerUrl
: 'http://localhost:11235';
const client = axios_1.default.create({
baseURL,
timeout: 60000,
});
if (this.credentials.connectionMode === 'docker' && this.credentials.authenticationType) {
if (this.credentials.authenticationType === 'token' && this.credentials.apiToken) {
client.defaults.headers.common['Authorization'] = `Bearer ${this.credentials.apiToken}`;
}
else if (this.credentials.authenticationType === 'basic' && this.credentials.username && this.credentials.password) {
const auth = Buffer.from(`${this.credentials.username}:${this.credentials.password}`).toString('base64');
client.defaults.headers.common['Authorization'] = `Basic ${auth}`;
}
}
client.interceptors.request.use((config) => {
console.log('Crawl4AI Request:', {
url: config.url,
method: config.method,
data: config.data,
});
return config;
}, (error) => {
console.error('Request error:', error);
return Promise.reject(error);
});
client.interceptors.response.use((response) => {
console.log('Crawl4AI Response:', {
status: response.status,
data: response.data,
});
return response;
}, (error) => {
var _a;
console.error('Response error:', ((_a = error.response) === null || _a === void 0 ? void 0 : _a.data) || error.message);
return Promise.reject(error);
});
return client;
}
async crawlUrl(url, config) {
try {
const response = await this.apiClient.post('/crawl', {
urls: [url],
browser_config: this.formatBrowserConfig(config),
crawler_config: this.formatCrawlerConfig(config),
});
if (response.data && Array.isArray(response.data.results) && response.data.results.length > 0) {
return response.data.results[0];
}
return {
url,
success: false,
error_message: 'No result returned from Crawl4AI API',
};
}
catch (error) {
console.error('Error during Crawl4AI API call:', error);
return {
url,
success: false,
error_message: error instanceof Error ? error.message : 'Unknown error occurred',
};
}
}
async crawlMultipleUrls(urls, config) {
try {
const response = await this.apiClient.post('/crawl', {
urls,
browser_config: this.formatBrowserConfig(config),
crawler_config: this.formatCrawlerConfig(config),
});
if (response.data && Array.isArray(response.data.results)) {
return response.data.results;
}
return urls.map(url => ({
url,
success: false,
error_message: 'No results returned from Crawl4AI API',
}));
}
catch (error) {
console.error('Error during Crawl4AI API call:', error);
return urls.map(url => ({
url,
success: false,
error_message: error instanceof Error ? error.message : 'Unknown error occurred',
}));
}
}
async processRawHtml(html, baseUrl, config) {
try {
const rawUrl = `raw://${html}`;
const response = await this.apiClient.post('/crawl', {
urls: [rawUrl],
browser_config: this.formatBrowserConfig(config),
crawler_config: {
...this.formatCrawlerConfig(config),
base_url: baseUrl,
},
});
if (response.data && Array.isArray(response.data.results) && response.data.results.length > 0) {
return response.data.results[0];
}
return {
url: baseUrl,
success: false,
error_message: 'No result returned from Crawl4AI API',
};
}
catch (error) {
console.error('Error during Crawl4AI API call:', error);
return {
url: baseUrl,
success: false,
error_message: error instanceof Error ? error.message : 'Unknown error occurred',
};
}
}
async arun(url, options) {
var _a, _b, _c;
try {
const crawlerConfig = {
cache_mode: options.cacheMode || 'enabled',
js_code: options.jsCode,
css_selector: options.cssSelector,
};
if (options.extractionStrategy) {
crawlerConfig.extraction_strategy = options.extractionStrategy;
}
if (options.extraArgs) {
Object.assign(crawlerConfig, options.extraArgs);
}
const requestBody = {
urls: [url],
browser_config: this.formatBrowserConfig(options.browserConfig || {}),
crawler_config: {
type: 'CrawlerRunConfig',
params: crawlerConfig,
},
};
console.log('Full request body:', JSON.stringify(requestBody, null, 2));
const response = await this.apiClient.post('/crawl', requestBody);
if (response.data && Array.isArray(response.data.results) && response.data.results.length > 0) {
return response.data.results[0];
}
return {
url,
success: false,
error_message: 'No result returned from Crawl4AI API',
};
}
catch (error) {
console.error('Error during Crawl4AI API call:', error);
console.error('Error response:', (_a = error.response) === null || _a === void 0 ? void 0 : _a.data);
return {
url,
success: false,
error_message: ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.data) === null || _c === void 0 ? void 0 : _c.detail) || error.message || 'Unknown error occurred',
};
}
}
formatBrowserConfig(config) {
return {
type: 'BrowserConfig',
params: {
headless: config.headless !== false,
java_script_enabled: config.javaScriptEnabled !== false,
viewport: config.viewport ? {
type: 'dict',
value: config.viewport,
} : { type: 'dict', value: { width: 1280, height: 800 } },
timeout: config.timeout || 30000,
user_agent: config.userAgent,
},
};
}
formatCrawlerConfig(config) {
const params = {
cache_mode: config.cacheMode || 'enabled',
stream: config.streamEnabled || false,
page_timeout: config.pageTimeout || 30000,
request_timeout: config.requestTimeout || 30000,
js_code: config.jsCode,
js_only: config.jsOnly || false,
css_selector: config.cssSelector,
excluded_tags: config.excludedTags || [],
exclude_external_links: config.excludeExternalLinks || false,
check_robots_txt: config.checkRobotsTxt || false,
word_count_threshold: config.wordCountThreshold || 0,
session_id: config.sessionId,
max_retries: config.maxRetries || 3,
};
if (config.extractionStrategy) {
params.extraction_strategy = config.extractionStrategy;
}
return {
type: 'CrawlerRunConfig',
params,
};
}
async close() {
}
}
exports.Crawl4aiClient = Crawl4aiClient;
async function createCrawlerInstance(credentials) {
return new Crawl4aiClient(credentials);
}
//# sourceMappingURL=apiClient.js.map