webforai
Version:
A library that provides a web interface for AI
65 lines (64 loc) • 2.79 kB
JavaScript
// src/extractors/presets/minimal-filter.ts
import { select } from "hast-util-select";
import { toString as hastToString } from "hast-util-to-string";
import { filter } from "unist-util-filter";
import { classnames, isStrInclude, matchString } from "./utils.js";
var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
var REGEXPS = {
hidden: /hidden|invisible|fallback-image/i,
byline: /byline|author|dateline|writtenby|p-author/i,
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
};
var metadataFilter = (node) => {
return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
};
var universalElementFilter = (node) => {
if (node.type !== "element") {
return true;
}
const element = node;
if (["aside", "nav"].includes(element.tagName)) {
return false;
}
if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
return false;
}
if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
return false;
}
if (element.tagName === "dialog") {
return false;
}
if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
return false;
}
if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
return false;
}
if (REGEXPS.byline.test(matchString(element))) {
return false;
}
if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
return false;
}
return true;
};
var minimalFilter = (params) => {
const { hast } = params;
const body = select("body", hast) ?? hast;
const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
if (!(metadataFilteredHast && metadataFilteredHastText)) {
return body;
}
const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3;
const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
return baseTree;
};
export {
minimalFilter
};