n8n-nodes-html-cleaner
Version:
Html Cleaner Node for n8n
328 lines • 15.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HtmlCleaner = void 0;
const n8n_workflow_1 = require("n8n-workflow");
const readability_1 = require("@mozilla/readability");
const cheerio = __importStar(require("cheerio"));
const jsdom_1 = require("jsdom");
const turndown_1 = __importDefault(require("turndown"));
class HtmlCleaner {
constructor() {
this.description = {
displayName: 'HTML Cleaner',
name: 'htmlCleaner',
icon: 'file:icon.svg',
group: ['transform'],
version: 1,
description: 'Cleans HTML content',
defaults: {
name: 'HTML Cleaner',
},
inputs: ["main"],
outputs: ["main"],
properties: [
{
displayName: 'HTML Content',
name: 'htmlContent',
type: 'string',
default: '',
placeholder: '<p>Enter HTML content here</p>',
description: 'The HTML content to clean',
},
{
displayName: 'Clean Options',
name: 'cleanOptions',
default: {},
type: 'collection',
options: [
{
displayName: 'Excluded Attributes',
name: 'excludedAttributes',
type: 'string',
default: '',
placeholder: 'onclick, onerror',
description: 'Comma-separated list of HTML attributes to exclude from cleaning. These attributes will be removed from the HTML tags.',
},
{
displayName: 'Excluded Selectors',
name: 'excludedSelectors',
type: 'string',
default: '',
placeholder: '.no-clean, #ignore',
description: 'Comma-separated list of CSS selectors to exclude from cleaning. These selectors will be removed from the HTML content.',
},
{
displayName: 'Excluded Tags',
name: 'excludedTags',
type: 'string',
default: '',
placeholder: 'script, style, iframe, nav, footer, aside, link, head, noscript, ul, svg, figure',
description: 'Comma-separated list of HTML tags to exclude from cleaning. These tags will be removed from the HTML content.',
},
{
displayName: 'Remove Attributes',
name: 'removeAttributes',
type: 'boolean',
default: true,
description: 'Whether to remove all attributes from HTML tags',
},
{
displayName: 'Remove Comments',
name: 'removeComments',
type: 'boolean',
default: true,
description: 'Whether to remove HTML comments',
},
{
displayName: 'Remove Empty Tags',
name: 'removeEmptyTags',
type: 'boolean',
default: true,
description: 'Whether to remove empty HTML tags',
},
{
displayName: 'Remove Scripts',
name: 'removeScripts',
type: 'boolean',
default: true,
description: 'Whether to remove script tags and their content',
},
{
displayName: 'Remove Styles',
name: 'removeStyles',
type: 'boolean',
default: true,
description: 'Whether to remove style tags and their content',
},
],
},
{
displayName: 'Readability Options',
name: 'readabilityOptions',
type: 'collection',
placeholder: 'Add Option',
description: "Options for @mozilla/readability parsing",
default: {},
options: [
{
displayName: "Char Threshold",
name: 'charThreshold',
type: 'number',
default: 500,
description: 'The number of characters an article must have in order to return a result',
},
{
displayName: "Classes To Preserve",
name: 'classesToPreserve',
type: 'string',
default: '',
placeholder: 'class1, class2',
description: 'A comma-separated list of classes to preserve on HTML elements when the keepClasses options is set to false',
},
{
displayName: "Disable JSON-LD",
name: 'disableJSONLD',
type: 'boolean',
default: false,
description: 'Whether to disable JSON-LD extraction. when extracting page metadata, Readability gives precedence to Schema.org fields specified in the JSON-LD format. Set this option to true to skip JSON-LD parsing.',
},
{
displayName: "Keep Classes",
name: 'keepClasses',
type: 'boolean',
default: false,
description: 'Whether to preserve all classes on HTML elements. When set to false only classes specified in the classesToPreserve array are kept.',
},
{
displayName: "NB Top Candidates",
name: 'nbTopCandidates',
type: 'number',
default: 5,
description: 'The number of top candidates to consider when analysing how tight the competition is among candidates',
},
]
},
{
displayName: 'Markdown Output',
name: 'markdownOutput',
type: 'boolean',
default: false,
description: 'Whether to output the cleaned HTML as Markdown. If set to true, the output will be converted to Markdown format using the `turndown` library.',
}
],
};
}
async execute() {
const items = this.getInputData();
const output = [];
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const htmlContent = this.getNodeParameter('htmlContent', itemIndex, '');
if (!htmlContent) {
if (this.continueOnFail()) {
output.push({
json: {
error: `HTML content is required for item ${itemIndex + 1}`,
message: `HTML content is required for item ${itemIndex + 1}`,
},
pairedItem: {
item: itemIndex,
},
});
}
else {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `HTML content is required for item ${itemIndex + 1}`, { itemIndex: itemIndex });
}
}
const cleanOptions = this.getNodeParameter('cleanOptions', itemIndex, {});
if (Object.keys(cleanOptions).length === 0) {
cleanOptions.removeComments = false;
cleanOptions.removeEmptyTags = false;
cleanOptions.removeScripts = false;
cleanOptions.removeStyles = false;
cleanOptions.removeAttributes = false;
cleanOptions.excludedAttributes = '';
cleanOptions.excludedSelectors = '';
cleanOptions.excludedTags = '';
}
try {
const $ = cheerio.load(htmlContent);
if (cleanOptions.removeComments) {
$('*').each((_, element) => {
const comments = $(element).contents().filter((_, node) => node.type === 'comment');
comments.remove();
});
}
if (cleanOptions.removeEmptyTags) {
$('*').each((_, element) => {
if ($(element).is(':empty')) {
$(element).remove();
}
});
}
if (cleanOptions.removeScripts) {
$('script').each((_, element) => {
$(element).remove();
});
}
if (cleanOptions.removeStyles) {
$('style').each((_, element) => {
$(element).remove();
});
}
if (cleanOptions.removeAttributes) {
$('*').each((_, element) => {
const attributes = $(element).attr();
attributes && Object.keys(attributes).forEach(attrName => {
$(element).removeAttr(attrName);
});
});
}
if (!cleanOptions.removeAttributes && cleanOptions.excludedAttributes) {
const excludedAttributes = cleanOptions.excludedAttributes.split(',').map((attr) => attr.trim());
$('*').each((_, element) => {
excludedAttributes.forEach(attrName => {
$(element).removeAttr(attrName);
});
});
}
if (cleanOptions.excludedSelectors) {
const excludedSelectors = cleanOptions.excludedSelectors.split(',').map((selector) => selector.trim());
excludedSelectors.forEach((selector) => {
$(selector).remove();
});
}
if (cleanOptions.excludedTags) {
const excludedTags = cleanOptions.excludedTags.split(',').map((tag) => tag.trim());
excludedTags.forEach((tag) => {
$(tag).remove();
});
}
const readabilityOptions = this.getNodeParameter('readabilityOptions', itemIndex, {});
if (Object.keys(readabilityOptions).length === 0) {
readabilityOptions.charThreshold = 500;
readabilityOptions.classesToPreserve = '';
readabilityOptions.disableJSONLD = false;
readabilityOptions.keepClasses = false;
readabilityOptions.nbTopCandidates = 5;
}
if (readabilityOptions.classesToPreserve) {
if (typeof readabilityOptions.classesToPreserve === 'string') {
readabilityOptions.classesToPreserve = readabilityOptions.classesToPreserve.split(',').map((cls) => cls.trim());
}
}
let html = $.html().replace(/\s+/g, " ").trim();
let dom = new jsdom_1.JSDOM(html, {
contentType: 'text/html',
includeNodeLocations: true,
pretendToBeVisual: true,
});
let article = new readability_1.Readability(dom.window.document, readabilityOptions).parse();
let outputContent = {
html,
title: (article === null || article === void 0 ? void 0 : article.title) || null,
lang: (article === null || article === void 0 ? void 0 : article.lang) || null,
content: (article === null || article === void 0 ? void 0 : article.content) || null,
textContent: (article === null || article === void 0 ? void 0 : article.textContent) || null,
length: (article === null || article === void 0 ? void 0 : article.length) || null,
excerpt: (article === null || article === void 0 ? void 0 : article.excerpt) || null
};
const markdownOutput = this.getNodeParameter('markdownOutput', itemIndex, false);
if (markdownOutput) {
const turndownService = new turndown_1.default();
outputContent.markdown = turndownService.turndown((article === null || article === void 0 ? void 0 : article.content) || '');
}
output.push({
json: { ...outputContent },
pairedItem: { item: itemIndex }
});
}
catch (error) {
if (this.continueOnFail()) {
output.push({ json: { error: error.message }, pairedItem: { item: itemIndex } });
}
else {
throw error;
}
}
}
return this.prepareOutputData(output);
}
}
exports.HtmlCleaner = HtmlCleaner;
//# sourceMappingURL=HtmlCleaner.node.js.map