n8n-nodes-free-web-scrapping
Version:
Node n8n for searching and extracting content from web pages via DuckDuckGo and free scraping.
271 lines (270 loc) • 12.3 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.WebScraper = void 0;
const got_1 = __importDefault(require("got"));
class WebScraper {
constructor() {
this.description = {
displayName: 'Web Scraper (Free)',
name: 'webScraper',
icon: 'file:icon.svg',
group: ['transform'],
version: 1,
subtitle: '={{$parameter["operation"]}}',
description: 'Perform free DuckDuckGo searches or scrape pages for headings, paragraphs, and more (no API key)',
defaults: { name: 'Web Scraper' },
inputs: ['main'],
outputs: ['main'],
properties: [
// 1) Operation selector
{
displayName: 'Operation',
name: 'operation',
type: 'options',
options: [
{ name: 'Search URLs only', value: 'searchUrls' },
{ name: 'Extract single page', value: 'extractPage' },
{ name: 'Filter page content', value: 'filterContent' },
{ name: 'Query + Full Scrape', value: 'fullScrape' },
],
default: 'searchUrls',
description: 'What you want to do',
},
// 2) Search Query
{
displayName: 'Search Query',
name: 'query',
type: 'string',
default: '',
displayOptions: {
show: { operation: ['searchUrls', 'fullScrape'] },
},
description: 'Text to search on DuckDuckGo',
},
// 3) Maximum URLs
{
displayName: 'Maximum URLs',
name: 'maxUrls',
type: 'number',
default: 5,
typeOptions: { minValue: 1, maxValue: 20 },
displayOptions: {
show: { operation: ['searchUrls', 'fullScrape'] },
},
description: 'How many result links to process',
},
// 4) Single‐page URL
{
displayName: 'Page URL',
name: 'url',
type: 'string',
default: '',
displayOptions: {
show: { operation: ['extractPage', 'filterContent'] },
},
description: 'The URL of the page to scrape',
},
// 5) Filter Type
{
displayName: 'Filter Type',
name: 'filterType',
type: 'options',
options: [
{ name: 'Headings (h1,h2,h3)', value: 'headings' },
{ name: 'Paragraphs', value: 'paragraphs' },
{ name: 'Bold Text', value: 'bolds' },
{ name: 'All Content', value: 'all' },
],
default: 'all',
displayOptions: {
show: { operation: ['filterContent'] },
},
description: 'Which parts of the page to return',
},
],
};
}
async execute() {
// Ensure at least one item for "Execute Node"
let items = this.getInputData();
if (items.length === 0) {
items = [{ json: {} }];
}
const returnData = [];
// Helpers for cleaning and extracting HTML
const decode = (t) => t
.replace(/ /gi, ' ')
.replace(/&/gi, '&')
.replace(/"/gi, '"')
.replace(/'/gi, "'")
.replace(/</gi, '<')
.replace(/>/gi, '>');
const stripTags = (t) => decode(t
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<!--[\s\S]*?-->/g, '')
.replace(/<\/?[^>]+>/g, ' '))
.replace(/\s+/g, ' ')
.trim();
const firstMatch = (re, src) => {
const m = re.exec(src);
return m && m[1] ? m[1] : null;
};
const grabAll = (re, src) => {
const out = [];
let m;
re.lastIndex = 0;
while ((m = re.exec(src))) {
if (m[1])
out.push(stripTags(m[1]));
}
return out;
};
// Blacklist for common social/video domains
const blacklisted = [
'youtube.com', 'instagram.com', 'facebook.com', 'twitter.com',
'tiktok.com', 'snapchat.com', 'linkedin.com', 'pinterest.com',
'reddit.com', 'twitch.tv', 'vimeo.com', 'dailymotion.com',
'bilibili.com', 'youku.com',
];
const isBlacklisted = (u) => {
try {
const host = u.replace(/^https?:\/\//i, '').split(/[\/?#]/)[0].toLowerCase();
return blacklisted.some(d => host.includes(d));
}
catch {
return false;
}
};
// DuckDuckGo search + URL extractor
const fetchUrls = async (query, max) => {
const html = await (0, got_1.default)(`https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`, { headers: { 'User-Agent': 'Mozilla/5.0' } }).text();
const linkRe = /<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"/gi;
const urls = [];
let m;
while (urls.length < max && (m = linkRe.exec(html))) {
let href = m[1];
if (href.startsWith('//'))
href = 'https:' + href;
const rd = href.match(/[?&]uddg=([^&]+)/);
if (rd)
href = decodeURIComponent(rd[1]);
if (/^https?:\/\//i.test(href) && !isBlacklisted(href)) {
urls.push(href);
}
}
return urls;
};
// Main loop
for (let i = 0; i < items.length; i++) {
const operation = this.getNodeParameter('operation', i);
try {
// SEARCH URLs or FULL SCRAPE
if (operation === 'searchUrls' || operation === 'fullScrape') {
const query = this.getNodeParameter('query', i);
const maxUrls = this.getNodeParameter('maxUrls', i);
const urls = await fetchUrls(query, maxUrls);
if (!urls.length) {
returnData.push({
json: { query, urls, error: 'NoResults', type: 'NoResults' },
});
continue;
}
if (operation === 'searchUrls') {
returnData.push(...urls.map(u => ({ json: { url: u } })));
continue;
}
// FULL SCRAPE
for (const u of urls) {
try {
const html = await (0, got_1.default)(u, { timeout: { request: 5000 } }).text();
const title = stripTags(firstMatch(/<meta[^>]+property=['"]og:title['"][^>]+content=['"]([^'"]+)['"]/i, html)
|| firstMatch(/<title[^>]*>([^<]+)<\/title>/i, html)
|| '') || null;
const h1 = grabAll(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, html);
const paragraphs = grabAll(/<p[^>]*>([\s\S]*?)<\/p>/gi, html)
.filter(p => p.length > 40);
returnData.push({ json: { url: u, title, h1, paragraphs } });
}
catch (err) {
const e = err;
this.logger.error('fullScrape item error', { error: e.message });
returnData.push({
json: { url: u, error: e.message, type: e.name },
});
}
}
}
// EXTRACT SINGLE PAGE
else if (operation === 'extractPage') {
const u = this.getNodeParameter('url', i);
try {
const html = await (0, got_1.default)(u, { timeout: { request: 5000 } }).text();
const title = stripTags(firstMatch(/<title[^>]*>([^<]+)<\/title>/i, html) || '') || null;
const h1 = grabAll(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, html);
const paragraphs = grabAll(/<p[^>]*>([\s\S]*?)<\/p>/gi, html)
.filter(p => p.length > 40);
returnData.push({ json: { url: u, title, h1, paragraphs } });
}
catch (err) {
const e = err;
this.logger.error('extractPage error', { error: e.message });
returnData.push({
json: { url: u, error: e.message, type: e.name },
});
}
}
// FILTER CONTENT
else if (operation === 'filterContent') {
const u = this.getNodeParameter('url', i);
const kind = this.getNodeParameter('filterType', i);
try {
const html = await (0, got_1.default)(u, { timeout: { request: 5000 } }).text();
let data;
switch (kind) {
case 'headings':
data = {
h1: grabAll(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, html),
h2: grabAll(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, html),
h3: grabAll(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, html),
};
break;
case 'paragraphs':
data = grabAll(/<p[^>]*>([\s\S]*?)<\/p>/gi, html)
.filter(p => p.length > 40);
break;
case 'bolds':
data = grabAll(/<(?:b|strong)[^>]*>([\s\S]*?)<\/(?:b|strong)>/gi, html);
break;
default:
data = {
h1: grabAll(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, html),
paragraphs: grabAll(/<p[^>]*>([\s\S]*?)<\/p>/gi, html)
.filter(p => p.length > 40),
bolds: grabAll(/<(?:b|strong)[^>]*>([\s\S]*?)<\/(?:b|strong)>/gi, html),
};
}
returnData.push({ json: { url: u, filterType: kind, data } });
}
catch (err) {
const e = err;
this.logger.error('filterContent error', { error: e.message });
returnData.push({
json: { url: u, error: e.message, type: e.name },
});
}
}
}
catch (err) {
const e = err;
this.logger.error('Operation failure', { error: e.message });
returnData.push({ json: { error: e.message, type: e.name } });
}
}
return this.prepareOutputData(returnData);
}
}
exports.WebScraper = WebScraper;