webforai/dist/extractors/presets/takumi.js

A library that provides a web interface for AI

// src/extractors/presets/takumi.ts
import { select, selectAll } from "hast-util-select";
import { toString as hastToString } from "hast-util-to-string";
import { filter } from "unist-util-filter";
import { classnames, isStrInclude, matchString } from "./utils.js";
var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
var REGEXPS = {
  hidden: /hidden|invisible|fallback-image/i,
  byline: /byline|author|dateline|writtenby|p-author/i,
  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
};
var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
var CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
var TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
var BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
var metadataFilter = (node) => {
  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
};
var universalElementFilter = (node) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (["aside", "nav"].includes(element.tagName)) {
    return false;
  }
  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
    return false;
  }
  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
    return false;
  }
  if (element.tagName === "dialog") {
    return false;
  }
  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
    return false;
  }
  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
    return false;
  }
  if (REGEXPS.byline.test(matchString(element))) {
    return false;
  }
  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
    return false;
  }
  return true;
};
var unlikelyElementFilter = (node) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (CONTENTABLE_TAGS.includes(element.tagName)) {
    return true;
  }
  const match = matchString(element);
  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
    return false;
  }
  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
    return false;
  }
  return true;
};
var removeEmptyFilter = (node, _lang) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (PARAGRAPH_TAGS.includes(element.tagName)) {
    return true;
  }
  if (element.tagName === "img" && !element.properties.src) {
    return false;
  }
  if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
    return false;
  }
  return true;
};
var takumiExtractor = (params) => {
  const { hast, lang = "en" } = params;
  const body = select("body", hast) ?? hast;
  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
  if (!(metadataFilteredHast && metadataFilteredHastText)) {
    return body;
  }
  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
  if (baseText.length < minimalLength) {
    minimalLength = Math.max(0, baseText.length - 200);
  }
  let extractedTree = baseTree;
  let extractedText = baseText;
  for (const selector of BODY_SELECTORS) {
    const content = { type: "root", children: selectAll(selector, baseFilterd) };
    const contentText = hastToString(content);
    if (contentText.length < 25) {
      continue;
    }
    const links = selectAll("a", content);
    const linkText = links.map((link) => hastToString(link)).join("");
    const linkDensity = linkText.length / contentText.length;
    if (linkDensity > 0.4) {
      continue;
    }
    if (contentText.length > minimalLength) {
      extractedTree = content;
      extractedText = contentText;
      break;
    }
  }
  const finalFilteredTree = filter(extractedTree, (node) => {
    if (!removeEmptyFilter(node, lang)) {
      return false;
    }
    if (!unlikelyElementFilter(node)) {
      return false;
    }
    return true;
  });
  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
  return finalTree;
};
export {
  takumiExtractor
};