webforai/dist/cjs/extractors/presets/takumi.js

A library that provides a web interface for AI

"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
  for (var name in all)
    __defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
  if (from && typeof from === "object" || typeof from === "function") {
    for (let key of __getOwnPropNames(from))
      if (!__hasOwnProp.call(to, key) && key !== except)
        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
  }
  return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var takumi_exports = {};
__export(takumi_exports, {
  takumiExtractor: () => takumiExtractor
});
module.exports = __toCommonJS(takumi_exports);
var import_hast_util_select = require("hast-util-select");
var import_hast_util_to_string = require("hast-util-to-string");
var import_unist_util_filter = require("unist-util-filter");
var import_utils = require("./utils");
const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
const REGEXPS = {
  hidden: /hidden|invisible|fallback-image/i,
  byline: /byline|author|dateline|writtenby|p-author/i,
  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
};
const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
const CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
const TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
const BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
const metadataFilter = (node) => {
  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
};
const universalElementFilter = (node) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (["aside", "nav"].includes(element.tagName)) {
    return false;
  }
  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
    return false;
  }
  if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
    return false;
  }
  if (element.tagName === "dialog") {
    return false;
  }
  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
    return false;
  }
  if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
    return false;
  }
  if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
    return false;
  }
  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
    return false;
  }
  return true;
};
const unlikelyElementFilter = (node) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (CONTENTABLE_TAGS.includes(element.tagName)) {
    return true;
  }
  const match = (0, import_utils.matchString)(element);
  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
    return false;
  }
  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
    return false;
  }
  return true;
};
const removeEmptyFilter = (node, _lang) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (PARAGRAPH_TAGS.includes(element.tagName)) {
    return true;
  }
  if (element.tagName === "img" && !element.properties.src) {
    return false;
  }
  if (TEXTABLE_TAGS.includes(element.tagName) && (0, import_hast_util_to_string.toString)(element).length === 0) {
    return false;
  }
  return true;
};
const takumiExtractor = (params) => {
  const { hast, lang = "en" } = params;
  const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
  const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
  const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
  if (!(metadataFilteredHast && metadataFilteredHastText)) {
    return body;
  }
  const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
  const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
  if (baseText.length < minimalLength) {
    minimalLength = Math.max(0, baseText.length - 200);
  }
  let extractedTree = baseTree;
  let extractedText = baseText;
  for (const selector of BODY_SELECTORS) {
    const content = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
    const contentText = (0, import_hast_util_to_string.toString)(content);
    if (contentText.length < 25) {
      continue;
    }
    const links = (0, import_hast_util_select.selectAll)("a", content);
    const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
    const linkDensity = linkText.length / contentText.length;
    if (linkDensity > 0.4) {
      continue;
    }
    if (contentText.length > minimalLength) {
      extractedTree = content;
      extractedText = contentText;
      break;
    }
  }
  const finalFilteredTree = (0, import_unist_util_filter.filter)(extractedTree, (node) => {
    if (!removeEmptyFilter(node, lang)) {
      return false;
    }
    if (!unlikelyElementFilter(node)) {
      return false;
    }
    return true;
  });
  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
  return finalTree;
};
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
  takumiExtractor
});