@wabarc/cairn
Version:
Node package and CLI tool for saving web page as single HTML file
590 lines • 21.8 kB
JavaScript
;
/*
* Copyright 2023 Wayback Archiver. All rights reserved.
* Use of this source code is governed by the MIT
* license that can be found in the LICENSE file.
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HTML = void 0;
const cheerio_1 = __importDefault(require("cheerio"));
const css_1 = require("./css");
const uri_1 = require("./uri");
const utils_1 = require("./utils");
/**
* @see https://html.spec.whatwg.org/multipage/semantics.html
*/
class HTML {
constructor(opt = {}) {
this.opt = opt;
// private opt: Record<string, unknown>;
this.rx = {
lazyImageSrc: /^\s*\S+(jpg|jpeg|png|webp|gif)\S*\s*$/gm,
lazyImageSrcset: /(jpg|jpeg|png|webp|gif)\s+\d/gm,
// Exclude SVG, because SVG can have a meaningful image in under 133 bytes.
B64DataURL: /data:(?!(image\/svg\+xml)).*?(;(.*?)),/gm,
srcsetURL: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/gm,
};
this.opt = opt;
}
/**
* Process assets within webpage
*
* @param {Object} page if error will be thrown
* @return {cheerio.Cheerio} [cheerio.Cheerio] call .html() parse as html string
* @api public
*/
process(page) {
return __awaiter(this, void 0, void 0, function* () {
const { html, uri } = page;
if (typeof html !== 'string' || typeof uri !== 'string') {
(0, utils_1.err)('Cannot process webpage.');
}
const $ = cheerio_1.default.load(html, { decodeEntities: false });
// Prepare documents by doing these steps :
// - Set Content-Security-Policy to make sure no unwanted request happened
// - Apply configuration to documents
// - Replace all noscript to divs, to make it processed as well
// - Remove all comments in documents
// - Convert data-src and data-srcset attribute in lazy image to src and srcset
// - Convert relative URL into absolute URL
// - Remove subresources integrity attribute from links
// - Convert Open Graph Metadata
// - Set page charset as utf-8
// - Set page source url
this.setContentSecurityPolicy($);
this.applyConfiguration($);
this.convertNoScriptToDiv($, true);
this.removeComments($);
this.convertLazyImageAttrs($);
this.convertRelativeURLs($, uri);
this.removeLinkIntegrityAttr($);
this.convertOpenGraph($);
this.setCharset($);
this.setSource($, uri);
// Find all nodes which might has subresource.
// A node might has subresource if it fulfills one of these criteria :
// - It has inline style;
// - It's link for icon or stylesheets;
// - It's tag name is either style, img, picture, figure, video, audio, source, iframe or object;
const tags = 'link,style,script,iframe,embed,object,img,picture,figure,video,audio,source';
const rels = [
'icon',
'stylesheet',
'shortcut icon',
'mask-icon',
'apple-touch-icon-precomposed',
'apple-touch-icon',
];
const nodes = [];
$.root()
.find(tags)
.each((_, e) => {
const $elem = $(e);
const tagName = $elem.get(0).tagName;
if (typeof tagName !== 'string') {
return;
}
switch (tagName.toLowerCase()) {
case 'link': {
const rel = $elem.attr('rel');
if (typeof rel === 'string' && rels.includes(rel)) {
nodes.push(e);
}
break;
}
case 'style':
case 'script':
case 'iframe':
case 'embed':
case 'object':
case 'img':
case 'picture':
case 'figure':
case 'video':
case 'audio':
case 'source': {
nodes.push(e);
break;
}
}
});
for (const node of nodes) {
const $node = $(node);
const tagName = $node.get(0).tagName;
if ($node.attr('style') !== undefined) {
yield this.processStyleAttr($node, uri);
}
switch (tagName.toLowerCase()) {
case 'style': {
yield this.processStyleNode($node, uri);
break;
}
case 'link': {
yield this.processLinkNode($node, uri);
break;
}
case 'script': {
yield this.processScriptNode($node, uri);
break;
}
case 'iframe':
case 'embed':
case 'object': {
yield this.processEmbedNode($node, uri);
break;
}
case 'img':
case 'picture':
case 'figure':
case 'video':
case 'audio':
case 'source': {
yield this.processMediaNode($node, uri);
break;
}
}
}
// Revert the converted noscripts
this.revertConvertedNoScript($);
// return cheerio.Root
// handle html() function convert to html string.
return $ || null;
});
}
/**
* setContentSecurityPolicy prevent browsers from requesting any remote
* resources by setting Content-Security-Policy to only allow from
* inline element and data URL.
*
* @param {Document} $ cheerio.Root
* @api private
*/
setContentSecurityPolicy($) {
// Remove existing CSP
$('meta[http-equiv="Content-Security-Policy"]').remove();
const policies = ["default-src 'unsafe-inline' data:;", "connect-src 'none';"];
if (this.opt.disableJS === true) {
policies.push("script-src 'none';");
}
if (this.opt.disableCSS === true) {
policies.push("style-src 'none';");
}
if (this.opt.disableEmbeds === true) {
policies.push("frame-src 'none'; child-src 'none';");
}
if (this.opt.disableMedias === true) {
policies.push("image-src 'none'; media-src 'none';");
}
// Append the new CSP
for (const policy of policies) {
$('head').prepend(`<meta http-equiv="Content-Security-Policy" content="${policy}">`);
}
}
/**
* Removes or replace elements following the configuration.
*
* @param {Document} $ cheerio.Root
* @api private
*/
applyConfiguration($) {
if (this.opt.disableJS === true) {
// Remove script tags
$('script').remove();
// Remove links with javascript URL scheme
$('a[href*="javascript:"]').attr('href', '#');
// Convert noscript to div
this.convertNoScriptToDiv($, false);
}
if (this.opt.disableCSS === true) {
// Remove style tags
$('style').remove();
// Remove inline style
$('[style]').removeAttr('style');
}
if (this.opt.disableEmbeds === true) {
$('embed,object,iframe').remove();
}
if (this.opt.disableMedias === true) {
$('img,picture,figure,video,audio,source').remove();
}
}
/**
* Convert all noscript to div element.
*
* @param {Document} $ cheerio.Root
* @param {boolean} [markNewDiv] mark to noscript
* @api private
*/
convertNoScriptToDiv($, markNewDiv = false) {
if (markNewDiv) {
let $node;
$('noscript').each((_, e) => {
$node = $(e);
$node.get(0).tagName = 'div';
$node.attr('data-cairn-noscript', 'true');
});
}
else {
$('noscript').each((_, e) => ($(e).get(0).tagName = 'div'));
}
}
/**
* Find all comments in document then remove it.
*
* @param {Document} $ cheerio.Root
* @api private
*/
removeComments($) {
$.root()
.find('*')
.contents()
.filter((_, e) => {
return e.type === 'comment';
})
.remove();
}
/**
* Convert attributes data-src and data-srcset which often found
* in lazy-loaded images and pictures, into basic attribute
* src and srcset, so images that can be loaded without JS.
*
* @param {Document} $ cheerio.Root
* @api private
*/
convertLazyImageAttrs($) {
// Convert img attributes
let $e;
$('img,picture,figure').each((_, e) => {
$e = $(e);
const src = $e.attr('src');
const srcset = $e.attr('srcset');
const tagName = $e.get(0).tagName.toLowerCase();
// In some sites (e.g. Kotaku), they put 1px square image as data uri in
// the src attribute. So, here we check if the data uri is too short,
// just might as well remove it.
if (src !== undefined && src.length > 0 && this.rx.B64DataURL.test(src)) {
return;
}
// let srcCouldBeRemoved: boolean = false;
// todo
if ((src || srcset) && $e.attr('loading') === 'lazy') {
return;
}
for (const [attrName, attrVal] of Object.entries($e.get(0).attribs)) {
if (attrName === undefined || typeof attrVal !== 'string') {
continue;
}
if (['src', 'srcset'].includes(attrName.toLowerCase())) {
continue;
}
let copyTo = '';
if (this.rx.lazyImageSrcset.test(attrVal)) {
copyTo = 'srcset';
}
else if (this.rx.lazyImageSrc.test(attrVal)) {
copyTo = 'src';
}
if (copyTo === '' || !(0, utils_1.isValidURL)(attrVal)) {
continue;
}
if (['img', 'picture'].includes(tagName)) {
$(e).attr(copyTo, attrVal);
}
else if (tagName === 'figure' && $(e).children('img, picture').length === 0) {
const img = `<img ${copyTo}=${attrVal}>`;
$(e).append(img);
}
$(e).removeAttr(attrName);
}
});
}
/**
* Converts all relative URL in document into absolute URL.
* We do this for a, img, picture, figure, video, audio, source, link,
* embed, iframe and object.
*
* @param {Document} $ cheerio.Root
* @param {string} [url] original request url
* @api private
*/
convertRelativeURLs($, url) {
const allowList = [
'a',
'link',
'embed',
'script',
'iframe',
'object',
'img',
'picture',
'figure',
'video',
'audio',
'source',
];
const slugs = {
a: 'href',
link: 'href',
embed: 'src',
script: 'src',
iframe: 'src',
object: 'data',
};
const mediaList = ['img', 'picture', 'figure', 'video', 'audio', 'source'];
const convert = (node, attrName) => {
const oriURI = $(node).attr(attrName);
if (typeof oriURI === 'string') {
let newVal = (0, utils_1.createAbsoluteURL)(oriURI, url);
try {
newVal = decodeURI(newVal);
}
catch (_) { }
$(node).attr(attrName, newVal);
}
};
let $e;
$('*').each((_, e) => {
$e = $(e);
let tagName = $e.get(0).tagName;
if (typeof tagName !== 'string' || Object.entries($e.get(0).attribs).length === 0) {
return;
}
tagName = tagName.toLowerCase();
if (allowList.includes(tagName) === false) {
return;
}
if (slugs[tagName]) {
const attrName = slugs[tagName];
convert(e, attrName);
}
if (mediaList.includes(tagName)) {
convert(e, 'src');
convert(e, 'poster');
const srcset = $(e).attr('srcset');
if (typeof srcset === 'string') {
const newSrcset = (0, utils_1.createAbsoluteURL)(srcset, url);
try {
$(e).attr('srcset', decodeURI(newSrcset));
}
catch (_) {
return;
}
}
}
});
}
/**
* Removes integrity attributes from link tags.
*
* @param {Document} $ cheerio.Root
* @api private
*/
removeLinkIntegrityAttr($) {
$('link[integrity]').removeAttr('integrity');
}
/**
* Set og:title to title when it empty.
*
* @param {Document} $ cheerio.Root
* @api private
*/
convertOpenGraph($) {
const title = $('head > title').text().trim();
$('head > meta').each((_, e) => {
const $elem = $(e);
const attr = $elem.attr('property');
const content = $elem.attr('content');
if (attr && typeof attr === 'string' && attr.startsWith('og:')) {
// real property
const property = attr.substring(3);
if (!$elem.attr(property)) {
const meta = `<meta property="${property}" content="${content}"/>`;
$elem.parent().append(meta);
// replace title if it empty
if (title.length < 1 && property.toLowerCase() === 'title') {
$('head > title').remove();
$('head').prepend(`<title>${content}</title>`);
}
}
}
});
}
/**
* Set webpage charset as UTF-8
*
* @param {Document} $ cheerio.Root
* @api private
*/
setCharset($) {
// Remove existing charset in meta
$('meta[charset]').remove();
// Append the new charset
$('head').prepend(`<meta charset="utf-8">`);
}
/**
* Set source webpage url
*
* @param {Document} $ cheerio.Root
* @param {string} url
* @api private
*/
setSource($, url) {
// Append the source url meta
$('head').append(`<meta property="source:url" content="${url}">`);
}
processStyleAttr(node, baseURL = '') {
return __awaiter(this, void 0, void 0, function* () {
const style = node.attr('style');
if (!style || style.length === 0) {
return;
}
const newStyle = yield css_1.css.process(style, baseURL);
if (newStyle.length > 0) {
node.attr('style', newStyle);
}
return;
});
}
processStyleNode(node, baseURL = '') {
return __awaiter(this, void 0, void 0, function* () {
const style = node.html();
if (!style || style.length === 0) {
return;
}
const newStyle = yield css_1.css.process(style, baseURL);
if (newStyle.length > 0) {
node.html(newStyle);
}
return;
});
}
processLinkNode(node, baseURL = '') {
return __awaiter(this, void 0, void 0, function* () {
const href = node.attr('href');
if (!href || href.length === 0) {
return;
}
const rel = node.attr('rel');
if (!rel || rel.length === 0) {
return;
}
if (rel.indexOf('icon') > -1) {
return yield this.processURLNode(node, 'href', baseURL);
}
// Replace <link> to <style>
if (['preload', 'stylesheet'].includes(rel.toLowerCase())) {
yield uri_1.uri.process(href, baseURL).then((data) => {
node.replaceWith(`<style type="text/css">${data}</style>`);
});
}
return;
});
}
processURLNode(node, attrName, baseURL) {
return __awaiter(this, void 0, void 0, function* () {
const url = node.attr(attrName);
if (typeof url !== 'string' || url.trim().length < 1) {
return;
}
const assetURL = (0, utils_1.createAbsoluteURL)(url, baseURL);
yield (0, utils_1.convertToData)(assetURL).then((data) => {
if (data && typeof data === 'string' && data.trim().length > -1) {
node.attr(attrName, data);
}
});
return;
});
}
processScriptNode(node, baseURL) {
return __awaiter(this, void 0, void 0, function* () {
const src = node.attr('src');
if (!src || typeof src !== 'string' || src.trim().length < 1) {
return;
}
yield uri_1.uri.process(src, baseURL).then((data) => {
node.removeAttr('src');
node.text(data);
});
return;
});
}
processEmbedNode(node, baseURL) {
return __awaiter(this, void 0, void 0, function* () {
const attrName = node.get(0).tagName === 'object' ? 'data' : 'src';
const url = node.attr(attrName);
if (!url || typeof url !== 'string' || url.trim().length < 1) {
return;
}
const assetURL = (0, utils_1.createAbsoluteURL)(url, baseURL);
yield (0, utils_1.convertToData)(assetURL).then((data) => {
if (data && typeof data === 'string' && data.trim().length > -1) {
node.removeAttr(attrName);
node.attr(attrName, data);
}
});
return;
});
}
processMediaNode(node, baseURL) {
return __awaiter(this, void 0, void 0, function* () {
const src = node.attr('src');
if (src && typeof src === 'string' && src.trim().length > 0) {
yield this.processURLNode(node, 'src', baseURL);
}
const poster = node.attr('poster');
if (poster && typeof poster === 'string' && poster.trim().length > 0) {
yield this.processURLNode(node, 'poster', baseURL);
}
let srcset = node.attr('srcset');
if (!srcset || typeof srcset !== 'string' || srcset.trim().length < 1) {
return;
}
try {
srcset = decodeURI(srcset);
}
catch (_) { }
let newSets = [];
const matches = [...srcset.matchAll(this.rx.srcsetURL)];
for (const parts of matches) {
if (!parts[1] || typeof parts[1] !== 'string') {
continue;
}
let newSet = parts[1];
const assetURL = (0, utils_1.createAbsoluteURL)(parts[1], baseURL);
const data = yield (0, utils_1.convertToData)(assetURL);
if (typeof data === 'string' && data.length > -1) {
newSet = data;
}
newSet += parts[2] || '';
newSets.push(newSet);
}
node.attr('srcset', newSets.join(','));
newSets = [];
return;
});
}
revertConvertedNoScript($) {
let $e;
$('div').each((_, e) => {
$e = $(e);
if ($e.attr('data-cairn-noscript') === 'true') {
$e.get(0).tagName = 'noscript';
}
});
return;
}
}
exports.HTML = HTML;
//# sourceMappingURL=html.js.map