webforai/dist/bin.js

A library that provides a web interface for AI

#!/usr/bin/env node

// src/cli/bin.ts
import { program } from "commander";

// package.json
var package_default = {
  name: "webforai",
  version: "2.1.1",
  description: "A library that provides a web interface for AI",
  author: "inaridiy",
  license: "Apache-2.0",
  keywords: [
    "web",
    "ai",
    "html",
    "html2md",
    "markdown",
    "mdast",
    "hast"
  ],
  repository: {
    type: "git",
    url: "https://github.com/inaridiy/webforai.git"
  },
  homepage: "https://webforai.dev/",
  scripts: {
    "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
    clean: "rimraf dist",
    build: "pnpm clean && tsx build.ts && pnpm copy:package.cjs.json",
    typecheck: "tsc --noEmit",
    prerelease: "pnpm build",
    release: "np"
  },
  files: [
    "dist",
    "!dist/types/**/*.js"
  ],
  main: "dist/cjs/index.js",
  type: "module",
  module: "dist/index.js",
  types: "dist/types/index.d.ts",
  bin: "dist/bin.js",
  exports: {
    ".": {
      types: "./dist/types/index.d.ts",
      import: "./dist/index.js",
      require: "./dist/cjs/index.js"
    },
    "./types": {
      types: "./dist/types/index.d.ts",
      import: "./dist/index.js",
      require: "./dist/cjs/index.js"
    },
    "./loaders/playwright": {
      types: "./dist/types/loaders/playwright.d.ts",
      import: "./dist/loaders/playwright.js",
      require: "./dist/cjs/loaders/playwright.js"
    },
    "./loaders/fetch": {
      types: "./dist/types/loaders/fetch.d.ts",
      import: "./dist/loaders/fetch.js",
      require: "./dist/cjs/loaders/fetch.js"
    },
    "./loaders/cf-puppeteer": {
      types: "./dist/types/loaders/cf-puppeteer.d.ts",
      import: "./dist/loaders/cf-puppeteer.js",
      require: "./dist/cjs/loaders/cf-puppeteer.js"
    },
    "./loaders/puppeteer": {
      types: "./dist/types/loaders/puppeteer.d.ts",
      import: "./dist/loaders/puppeteer.js",
      require: "./dist/cjs/loaders/puppeteer.js"
    }
  },
  typesVersions: {
    "*": {
      types: [
        "./dist/types/index.d.ts"
      ],
      "loaders/playwright": [
        "./dist/types/loaders/playwright.d.ts"
      ],
      "loaders/cf-puppeteer": [
        "./dist/types/loaders/cf-puppeteer.d.ts"
      ],
      "loaders/fetch": [
        "./dist/types/loaders/fetch.d.ts"
      ],
      "loaders/puppeteer": [
        "./dist/types/loaders/puppeteer.d.ts"
      ]
    }
  },
  peerDependencies: {
    "@cloudflare/puppeteer": ">=0.0.6",
    "playwright-core": ">=1.4",
    puppeteer: ">=22"
  },
  peerDependenciesMeta: {
    "@cloudflare/puppeteer": {
      optional: true
    },
    "playwright-core": {
      optional: false
    },
    puppeteer: {
      optional: true
    }
  },
  dependencies: {
    "@clack/prompts": "^0.7.0",
    boxen: "^8.0.1",
    commander: "^12.1.0",
    "hast-util-from-html": "^2.0.3",
    "hast-util-select": "^6.0.2",
    "hast-util-to-html": "^9.0.3",
    "hast-util-to-mdast": "^10.1.0",
    "hast-util-to-string": "^3.0.0",
    "hast-util-to-text": "^4.0.0",
    "mathml-to-latex": "^1.4.1",
    "mdast-util-gfm": "^3.0.0",
    "mdast-util-math": "^3.0.0",
    "mdast-util-to-markdown": "^2.1.0",
    picocolors: "^1.0.1",
    "trim-trailing-lines": "^2.1.0",
    "unist-util-filter": "^5.0.1",
    zx: "^8.1.5"
  },
  devDependencies: {
    "@cloudflare/puppeteer": "^0.0.6",
    "@tsconfig/recommended": "^1.0.3",
    "@types/hast": "^3.0.2",
    "@types/mdast": "^4.0.2",
    "@types/node": "^20.14.10",
    arg: "^5.0.2",
    esbuild: "^0.19.11",
    "fastest-levenshtein": "^1.0.16",
    glob: "^10.3.10",
    ncp: "^2.0.0",
    np: "^9.2.0",
    "playwright-core": "^1.40.1",
    puppeteer: "^23.2.2",
    rimraf: "^5.0.5",
    tsx: "^4.19.1",
    typescript: "^5.4.5"
  }
};

// src/cli/commands/webforai/index.ts
import fs5 from "node:fs/promises";
import path2 from "node:path";
import { intro, log as log2, outro, spinner } from "@clack/prompts";
import pc2 from "picocolors";

// src/html-to-mdast.ts
import { fromHtml } from "hast-util-from-html";
import { toMdast } from "hast-util-to-mdast";

// src/extract-mdast.ts
import { filter } from "unist-util-filter";
var DECLATION_TYPES = ["blockquote", "strong", "emphasis", "delete"];
var emptyDeclarationFilter = (node) => {
  if (!DECLATION_TYPES.includes(node.type)) {
    return true;
  }
  if (node.children.length === 0) {
    return false;
  }
  return true;
};
var extractMdast = (node) => {
  const extracted = filter(node, (node2) => {
    if (!emptyDeclarationFilter(node2)) {
      return false;
    }
    return true;
  });
  return extracted;
};

// src/extractors/presets/takumi.ts
import { select, selectAll } from "hast-util-select";
import { toString as hastToString } from "hast-util-to-string";
import { filter as filter2 } from "unist-util-filter";

// src/extractors/presets/utils.ts
var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
var classnames = (element) => {
  if (Array.isArray(element.properties.className)) {
    return element.properties.className;
  }
  return [];
};
var isStrInclude = (value, match) => {
  if (typeof value === "string") {
    return value.includes(match);
  }
  return false;
};

// src/extractors/presets/takumi.ts
var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
var REGEXPS = {
  hidden: /hidden|invisible|fallback-image/i,
  byline: /byline|author|dateline|writtenby|p-author/i,
  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
};
var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
var CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
var TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
var BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
var metadataFilter = (node) => {
  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
};
var universalElementFilter = (node) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (["aside", "nav"].includes(element.tagName)) {
    return false;
  }
  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
    return false;
  }
  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
    return false;
  }
  if (element.tagName === "dialog") {
    return false;
  }
  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
    return false;
  }
  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
    return false;
  }
  if (REGEXPS.byline.test(matchString(element))) {
    return false;
  }
  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
    return false;
  }
  return true;
};
var unlikelyElementFilter = (node) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (CONTENTABLE_TAGS.includes(element.tagName)) {
    return true;
  }
  const match = matchString(element);
  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
    return false;
  }
  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
    return false;
  }
  return true;
};
var removeEmptyFilter = (node, _lang) => {
  if (node.type !== "element") {
    return true;
  }
  const element = node;
  if (PARAGRAPH_TAGS.includes(element.tagName)) {
    return true;
  }
  if (element.tagName === "img" && !element.properties.src) {
    return false;
  }
  if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
    return false;
  }
  return true;
};
var takumiExtractor = (params) => {
  const { hast, lang = "en" } = params;
  const body = select("body", hast) ?? hast;
  const metadataFilteredHast = filter2(body, (node) => metadataFilter(node));
  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
  if (!(metadataFilteredHast && metadataFilteredHastText)) {
    return body;
  }
  const baseFilterd = filter2(metadataFilteredHast, (node) => universalElementFilter(node));
  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
  if (baseText.length < minimalLength) {
    minimalLength = Math.max(0, baseText.length - 200);
  }
  let extractedTree = baseTree;
  let extractedText = baseText;
  for (const selector of BODY_SELECTORS) {
    const content = { type: "root", children: selectAll(selector, baseFilterd) };
    const contentText = hastToString(content);
    if (contentText.length < 25) {
      continue;
    }
    const links = selectAll("a", content);
    const linkText = links.map((link) => hastToString(link)).join("");
    const linkDensity = linkText.length / contentText.length;
    if (linkDensity > 0.4) {
      continue;
    }
    if (contentText.length > minimalLength) {
      extractedTree = content;
      extractedText = contentText;
      break;
    }
  }
  const finalFilteredTree = filter2(extractedTree, (node) => {
    if (!removeEmptyFilter(node, lang)) {
      return false;
    }
    if (!unlikelyElementFilter(node)) {
      return false;
    }
    return true;
  });
  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
  return finalTree;
};

// src/constants.ts
var DEFAULT_EXTRACTORS = [takumiExtractor];

// src/extractors/pipeExtractors.ts
var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
  const { hast, lang } = params;
  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
  const extracted = _extractors.reduce((acc, extractor) => {
    if (extractor === false) {
      return acc;
    }
    if (typeof extractor === "function") {
      return extractor({ hast: acc, lang });
    }
    throw new Error(`Invalid extractor: ${extractor}`);
  }, hast) || hast;
  return extracted;
};

// src/extractors/presets/minimal-filter.ts
import { select as select2 } from "hast-util-select";
import { toString as hastToString2 } from "hast-util-to-string";
import { filter as filter3 } from "unist-util-filter";

// src/mdast-handlers/custom-a-handler.ts
import { defaultHandlers } from "hast-util-to-mdast";
import { toString as hastToString3 } from "hast-util-to-string";
var customAHandler = (options) => (state, node) => {
  if (options?.asText) {
    const text3 = hastToString3(node);
    if (3 >= text3.length) {
      return void 0;
    }
    const link2 = { type: "text", value: text3 };
    state.patch(node, link2);
    return link2;
  }
  const link = defaultHandlers.a(state, node);
  if (link.children.length > 0) {
    return link;
  }
  return void 0;
};

// src/mdast-handlers/custom-code-handler.ts
import { toText } from "hast-util-to-text";
import { trimTrailingLines } from "trim-trailing-lines";

// src/utils/detect-code-lang.ts
var languages = [
  ["bash", [/#!(\/usr)?\/bin\/bash/g, 500], [/\b(if|elif|then|fi|echo)\b|\$/g, 10]],
  ["html", [/<\/?[a-z-]+[^\n>]*>/g, 10], [/^\s+<!DOCTYPE\s+html/g, 500]],
  ["http", [/^(GET|HEAD|POST|PUT|DELETE|PATCH|HTTP)\b/g, 500]],
  [
    "ts",
    [/(import .* from|export \*|export const|const .* = await)/g, 400],
    [
      /\b(console|await|async|function|undefined|export|import|this|class|for|let|const|map|join|require|implements|interface|namespace)\b/g,
      10
    ]
  ],
  [
    "tsx",
    [/(import .* from|export \*|export const)/g, 400],
    [/\b(react|next|FC)\b/g, 200],
    [
      /\b(console|await|async|function|export|undefined|import|this|class|for|let|const|map|join|require|implements|interface)\b/g,
      10
    ]
  ],
  ["py", [/\b(def|print|class|and|or|lambda)\b/g, 10]],
  ["sql", [/\b(SELECT|INSERT|FROM)\b/g, 50]],
  ["pl", [/#!(\/usr)?\/bin\/perl/g, 500], [/\b(use|print)\b|\$/g, 10]],
  ["lua", [/#!(\/usr)?\/bin\/lua/g, 500]],
  ["make", [/\b(ifneq|endif|if|elif|then|fi|echo|.PHONY|^[a-z]+ ?:$)\b|\$/gm, 10]],
  ["uri", [/https?:|mailto:|tel:|ftp:/g, 30]],
  ["css", [/^(@import|@page|@media|(\.|#)[a-z]+)/gm, 20]],
  ["diff", [/^[+><-]/gm, 10], [/^@@ ?[-+,0-9 ]+ ?@@/gm, 25]],
  ["md", [/^(>|\t\*|\t\d+.)/gm, 10], [/\[.*\](.*)/g, 10]],
  ["docker", [/^(FROM|ENTRYPOINT|RUN)/gm, 500]],
  ["xml", [/<\/?[a-z-]+[^\n>]*>/g, 10], [/^<\?xml/g, 500]],
  ["c", [/#include\b|\bprintf\s+\(/g, 100]],
  ["rs", [/^\s+(use|fn|mut|match)\b/gm, 100]],
  ["go", [/\b(func|fmt|package)\b/g, 100]],
  ["java", [/^import\s+java/gm, 500]],
  ["asm", [/^(section|global main|extern|\t(call|mov|ret))/gm, 100]],
  ["css", [/^(@import|@page|@media|(\.|#)[a-z]+)/gm, 20]],
  ["json", [/\b(true|false|null|\{})\b|\"[^"]+\":/g, 10]],
  ["yaml", [/^(\s+)?[a-z][a-z0-9]*:/gim, 10]]
];
var detectLanguage = (code) => {
  return languages.map(
    ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
  ).filter(([_, score]) => score > 20).sort((a, b) => b[1] - a[1])[0]?.[0] || "plain";
};

// src/mdast-handlers/custom-code-handler.ts
var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
var customCodeHandler = (state, node) => {
  const classNames = node.properties?.className || [];
  const codeValue = trimTrailingLines(toText(node)).trim();
  const classLang = classNames.map((className) => {
    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
    return match?.[1];
  }).find((className) => className);
  const lang = classLang || detectLanguage(codeValue) || null;
  const result = { type: "code", lang, meta: null, value: codeValue };
  state.patch(node, result);
  return result;
};

// src/mdast-handlers/custom-div-handler.ts
import { select as select3 } from "hast-util-select";
import { defaultHandlers as defaultHandlers2 } from "hast-util-to-mdast";
import { toString as hastToString4 } from "hast-util-to-string";
import { toText as toText2 } from "hast-util-to-text";
import { trimTrailingLines as trimTrailingLines2 } from "trim-trailing-lines";
var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
var LANGUAGE_MATCH_REGEX2 = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
var findRecursive = (array, condition, maxDepth = 3) => {
  if (maxDepth <= 0) {
    return null;
  }
  for (const value of array) {
    const result = condition(value);
    if (Array.isArray(result)) {
      return findRecursive(result, condition, maxDepth - 1);
    }
    if (result) {
      return value;
    }
  }
  return null;
};
var customDivHandler = (state, node) => {
  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
  const codeBlock = findRecursive(node.children, (child) => {
    if (child.type !== "element") {
      return false;
    }
    if (child.tagName === "pre") {
      return true;
    }
    return child.children.filter((child2) => child2.type === "element");
  });
  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
    const codeValue = trimTrailingLines2(toText2(codeBlock)).trim();
    const filenameElement = select3(CODE_FILENAME_SELECTORS, node);
    const fileLang = filenameElement ? hastToString4(filenameElement).match(/\.(\w+)$/)?.[1] : null;
    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
      const match = LANGUAGE_MATCH_REGEX2.map((regex) => className.match(regex)).find((match2) => match2);
      return match?.[1];
    }).find((className) => className);
    const lang = fileLang || classLang || detectLanguage(codeValue) || null;
    const result = { type: "code", lang, meta: null, value: codeValue };
    state.patch(node, result);
    return result;
  }
  return defaultHandlers2.div(state, node);
};

// src/mdast-handlers/custom-img-handler.ts
import { defaultHandlers as defaultHandlers3 } from "hast-util-to-mdast";
var customImgHandler = (options) => (state, node) => {
  if (options?.hideImage) {
    return void 0;
  }
  return defaultHandlers3.image(state, node);
};

// src/mdast-handlers/custom-table-handler.ts
import { defaultHandlers as defaultHandlers4 } from "hast-util-to-mdast";
import { toText as toText3 } from "hast-util-to-text";
var customTableHandler = (options) => (state, node) => {
  if (options?.asText) {
    const paragraph = { type: "paragraph", children: [{ type: "text", value: toText3(node) }] };
    state.patch(node, paragraph);
    return paragraph;
  }
  return defaultHandlers4.table(state, node);
};

// src/mdast-handlers/math-handler.ts
import { toHtml } from "hast-util-to-html";
import { MathMLToLaTeX } from "mathml-to-latex";
var mathHandler = (state, node) => {
  const mathMl = toHtml(node);
  const latex = MathMLToLaTeX.convert(mathMl);
  const result = { type: "inlineMath", value: latex };
  state.patch(node, result);
  return result;
};

// src/utils/hast-utils.ts
import { select as select4, selectAll as selectAll2 } from "hast-util-select";
var getLangFromHast = (node) => {
  const html = select4("html", node);
  if (html && typeof html.properties.lang === "string") {
    return html.properties.lang;
  }
  if (node.type !== "element") {
    return;
  }
  const element = node;
  if (element.tagName !== "html") {
    return;
  }
  const langAttr = element.properties.lang || element.properties["xml:lang"];
  if (langAttr) {
    return langAttr;
  }
  return void 0;
};
var getLangFromStr = (str) => {
  const match = str.match(/lang=["']([^"']+)["']/);
  if (match) {
    return match[1];
  }
  return void 0;
};
var getUrlFromHast = (node) => {
  if (node.type !== "element") {
    return void 0;
  }
  const metaTagAttributes = ["og:url", "twitter:url"];
  const metaTags = selectAll2("meta", node);
  for (const meta of metaTags) {
    const property = meta.properties.property || meta.properties.name;
    if (typeof property === "string" && metaTagAttributes.includes(property)) {
      return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
    }
  }
  return void 0;
};

// src/html-to-mdast.ts
var htmlToMdast = (htmlOrHast, options) => {
  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
  const url = defaultUrl || getUrlFromHast(hast);
  const extractedHast = pipeExtractors({ hast, lang, url }, extractors);
  const mdast = toMdast(extractedHast, {
    handlers: {
      math: mathHandler,
      div: customDivHandler,
      pre: customCodeHandler,
      a: customAHandler({ asText: options?.linkAsText }),
      img: customImgHandler({ hideImage: options?.hideImage }),
      table: customTableHandler({ asText: options?.tableAsText })
    }
  });
  const extractedMdast = extractMdast(mdast);
  return extractedMdast;
};

// src/mdast-to-markdown.ts
import { gfmToMarkdown } from "mdast-util-gfm";
import { mathToMarkdown } from "mdast-util-math";
import { toMarkdown } from "mdast-util-to-markdown";

// src/link-replacer.ts
var linkReplacer = (markdown, base) => {
  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
  return markdown.replace(regex, (match, pre, url, post) => {
    if (/^(https?:|#)/.test(url)) {
      return match;
    }
    try {
      const absoluteUrl = new URL(url, base).href;
      return `${pre}${absoluteUrl}${post}`;
    } catch {
      return match;
    }
  });
};

// src/utils/mdast-utils.ts
var warpRoot = (mdast) => {
  if (Array.isArray(mdast)) {
    return { type: "root", children: mdast };
  }
  return mdast;
};

// src/mdast-to-markdown.ts
var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
  extensions: [gfmToMarkdown(), mathToMarkdown()],
  bullet: "-"
};
var mdastToMarkdown = (mdast, options) => {
  const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
  if (baseUrl) {
    markdown = linkReplacer(markdown, baseUrl);
  }
  return markdown;
};

// src/html-to-markdown.ts
var htmlToMarkdown = (htmlOrHast, options) => {
  const { baseUrl, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
  const mdast = htmlToMdast(htmlOrHast, toMdastOptions);
  const markdown = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
  return markdown;
};

// src/cli/helpers/inputOutputPath.ts
import fs2 from "node:fs";
import { confirm, text } from "@clack/prompts";

// src/cli/utils.ts
import fs from "node:fs";
import path from "node:path";
var isUrl = (maybeUrl) => {
  try {
    new URL(maybeUrl);
    return true;
  } catch {
    return false;
  }
};
function changeFileExtension(filePath, newExtension) {
  const parsedPath = filePath.split("/");
  const fileName = parsedPath[parsedPath.length - 1];
  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
  if (fileName.startsWith(".")) {
    const parts = fileName.split(".");
    if (parts.length === 2) {
      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
    }
    parts[parts.length - 1] = newExtension.replace(/^\./, "");
    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
  }
  const lastDotIndex = fileName.lastIndexOf(".");
  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
  const newFileName = `${baseName}${formattedNewExtension}`;
  parsedPath[parsedPath.length - 1] = newFileName;
  return parsedPath.join("/");
}
function urlToFilename(url) {
  try {
    const urlObj = new URL(url);
    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
      if (index === 0) {
        return acc;
      }
      if (acc.length >= 2) {
        return acc;
      }
      if (part === "www") {
        return acc;
      }
      return [part, ...acc];
    }, []);
    const domainString = domainParts.reverse().join("-");
    const pathParts = urlObj.pathname.split("/").filter(Boolean);
    const relevantPathParts = pathParts.slice(-2);
    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
    let filename = [domainString, pathString].filter(Boolean).join("-");
    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
    return filename || "output";
  } catch {
    return "output";
  }
}
var sourcePathToOutputPath = (sourcePath) => {
  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
};
function getNextAvailableFilePath(filePath) {
  const parsedPath = path.parse(filePath);
  const directory = parsedPath.dir;
  const fullName = parsedPath.base;
  const [firstPart, ...restParts] = fullName.split(".");
  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
  const baseName = firstPart.replace(/_\d+$/, "");
  let counter = 1;
  let nextFilePath = filePath;
  while (fs.existsSync(nextFilePath)) {
    const match = firstPart.match(/_(\d+)$/);
    if (match) {
      counter = Number.parseInt(match[1], 10) + 1;
    }
    const newName = `${baseName}_${counter}${restName}`;
    nextFilePath = path.join(directory, newName);
    counter++;
  }
  return nextFilePath;
}

// src/cli/helpers/assertContinue.ts
import { cancel, isCancel } from "@clack/prompts";
function assertContinue(message, cancelMessage = "Canceled.") {
  if (isCancel(message)) {
    cancel(cancelMessage);
    process.exit(1);
  }
}

// src/cli/helpers/inputOutputPath.ts
var inputOutputPath = async (sourcePath) => {
  const outputPath = await text({
    message: "Enter the output file path:",
    placeholder: "output.md",
    initialValue: sourcePathToOutputPath(sourcePath),
    validate: (value) => {
      if (value.trim() === "") {
        return "Output path is required";
      }
      if (fs2.existsSync(value) && fs2.statSync(value).isDirectory()) {
        return "No directory can be specified.";
      }
      if (!fs2.existsSync(value) && value.endsWith("/")) {
        return "No directory can be specified.";
      }
    }
  });
  assertContinue(outputPath);
  if (!fs2.existsSync(outputPath)) {
    return outputPath;
  }
  const isOutputFileOverwrite = await confirm({
    message: "The file already exists. Overwrite?",
    initialValue: false
  });
  assertContinue(isOutputFileOverwrite);
  if (isOutputFileOverwrite) {
    return outputPath;
  }
  const escapedOutputPath = await text({
    message: "Enter the output file path:",
    placeholder: "output.md",
    initialValue: getNextAvailableFilePath(outputPath),
    validate: (value) => {
      if (value.trim() === "") {
        return "Output path is required";
      }
      if (fs2.existsSync(value)) {
        return "The file already exists";
      }
      if (!fs2.existsSync(value) && value.endsWith("/")) {
        return "No directory can be specified.";
      }
    }
  });
  assertContinue(escapedOutputPath);
  return escapedOutputPath;
};

// src/cli/helpers/inputSourcePath.ts
import fs3 from "node:fs";
import { text as text2 } from "@clack/prompts";

// src/cli/constants.ts
var DEFAULT_PATH = "https://example.com";
var LOADERS = ["fetch", "playwright"];
var MODES = ["default", "ai"];

// src/cli/helpers/inputSourcePath.ts
var inputSourcePath = async () => {
  const result = await text2({
    message: "Enter the URL or html path to be converted to markdown:",
    placeholder: DEFAULT_PATH,
    initialValue: "",
    validate: (value) => {
      if (value.trim() === "") {
        return "Source is required";
      }
      if (!isUrl(value)) {
        if (!fs3.existsSync(value)) {
          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
        }
        if (fs3.statSync(value).isDirectory()) {
          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
        }
      }
    }
  });
  assertContinue(result);
  return result;
};

// src/cli/helpers/selectExtractMode.ts
import { select as select5 } from "@clack/prompts";
var selectExtractMode = async () => {
  const result = await select5({
    message: "Select processing mode:",
    options: MODES.map((mode) => ({ value: mode, label: mode })),
    initialValue: MODES[0]
  });
  assertContinue(result);
  return result;
};

// src/cli/helpers/selectLoader.ts
import { select as select6 } from "@clack/prompts";
var loadersHint = {
  fetch: "Fetch HTML content from the given URL",
  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
};
var selectLoader = async () => {
  const result = await select6({
    message: "Select loader:",
    initialValue: "fetch",
    options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
  });
  assertContinue(result);
  return result;
};

// src/cli/commands/webforai/loadHtml.ts
import fs4 from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { log } from "@clack/prompts";
import boxen from "boxen";
import pc from "picocolors";
import { chromium as chromium2 } from "playwright-core";

// src/loaders/fetch.ts
var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
var loadHtml = async (url, userAgent = USER_AGENT) => {
  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
  return response.text();
};

// src/loaders/playwright.ts
import { chromium, devices } from "playwright-core";
var SUPER_BYPASS_DEVICE = {
  userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
  viewport: { width: 1920, height: 1080 },
  deviceScaleFactor: 1,
  hasTouch: false,
  isMobile: false,
  javaScriptEnabled: true,
  locale: "en-US",
  timezoneId: "America/New_York"
};
var loadHtml2 = async (url, options) => {
  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
  const _browser = browser ?? await chromium.launch({ headless: true });
  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
  if (superBypassMode) {
    await context.addInitScript(() => {
      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
    });
  }
  const page = await context.newPage();
  if (superBypassMode) {
    await page.route("**/*.js", (route) => {
      if (route.request().url().includes("captcha-delivery")) {
        return route.abort();
      }
      return route.continue();
    });
  }
  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
  await page.evaluate(() => {
    const elements = document.querySelectorAll("*");
    for (const element of elements) {
      const rect = element.getBoundingClientRect();
      element.setAttribute("data-rwidth", rect.width.toString());
      element.setAttribute("data-rheight", rect.height.toString());
    }
  });
  const html = await page.content();
  await page.close();
  if (browser) {
    await context.close();
  } else {
    await _browser.close();
  }
  return html;
};

// src/cli/commands/webforai/loadHtml.ts
var checkPlaywrightAvailable = async () => {
  const path3 = chromium2.executablePath();
  try {
    await fs4.access(path3);
    return true;
  } catch {
    return false;
  }
};
var getPlaywrightVersion = async () => {
  const path3 = await import.meta.resolve("playwright-core/package.json");
  const pwPackageJson = await fs4.readFile(fileURLToPath(path3), "utf-8").then((res) => JSON.parse(res.toString())).catch(() => null);
  return pwPackageJson?.version;
};
var loadHtml3 = async (sourcePath, loader, options) => {
  if (loader === "local") {
    options.debug && log.info(`Loading HTML from local file: ${sourcePath}`);
    const content = await fs4.readFile(sourcePath, "utf-8");
    options.debug && log.info(`HTML loaded: ${content.slice(0, 100)}`);
    return content;
  }
  if (loader === "fetch") {
    options.debug && log.info(`Loading HTML from URL: ${sourcePath}`);
    const content = await loadHtml(sourcePath);
    options.debug && log.info(`HTML loaded: ${content.slice(0, 100)}`);
    return content;
  }
  if (loader === "playwright") {
    options.debug && log.info(`Loading HTML from playwright: ${sourcePath}`);
    const isPlaywrightAvailable = await checkPlaywrightAvailable();
    options.debug && log.info(`Playwright available: ${isPlaywrightAvailable}`);
    const pwVersion = await getPlaywrightVersion();
    if (!isPlaywrightAvailable) {
      const message = [
        pc.bold("Error: Playwright is not available"),
        "",
        "To use the Playwright loader, please install Playwright by running:",
        "",
        `  npx playwright@${pwVersion} install chromium`,
        "",
        "Hint 1: If you receive a warning like this:",
        `  "WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies."`,
        "You can safely ignore this warning.",
        "",
        "Hint 2: If you encounter the following message:",
        `  "Host system is missing dependencies to run browsers."`,
        "You should install the necessary dependencies by executing:",
        "",
        `  sudo npx playwright@${pwVersion} install-deps`
      ];
      log.error(boxen(message.join("\n"), { padding: 1, borderStyle: "round" }));
      throw new Error("Playwright is not available");
    }
    const content = await loadHtml2(sourcePath);
    options.debug && log.info(`HTML loaded: ${content.slice(0, 100)}`);
    return content;
  }
  throw new Error(`Unsupported loader: ${loader}`);
};

// src/cli/commands/webforai/index.ts
var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
var webforaiCommand = async (initialPath, options) => {
  intro(pc2.bold(pc2.green(`webforai CLI version ${package_default.version}`)));
  const sourcePath = initialPath ?? await inputSourcePath();
  options.debug && log2.info(`sourcePath: ${sourcePath}`);
  const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
  options.debug && log2.info(`loader: ${loader}`);
  const outputPath = options.output ?? await inputOutputPath(sourcePath);
  options.debug && log2.info(`outputPath: ${outputPath}`);
  const mode = options.mode ?? await selectExtractMode();
  options.debug && log2.info(`mode: ${mode}`);
  let html;
  const s = spinner();
  try {
    s.start("Loading content...");
    html = await loadHtml3(sourcePath, loader, { debug: options.debug });
    s.stop(pc2.green("Content loaded!"));
  } catch (error) {
    s.stop(pc2.red("Content loading failed!"));
    console.error(error);
    process.exit(1);
  }
  options.debug && log2.info(`html: ${html}`);
  const markdown = htmlToMarkdown(html, {
    baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
  });
  options.debug && log2.info(`markdown: ${markdown}`);
  const directory = path2.dirname(outputPath);
  const isDirectoryExists = await fs5.stat(directory).then((stat) => stat.isDirectory());
  if (!isDirectoryExists) {
    await fs5.mkdir(directory, { recursive: true });
  }
  await fs5.writeFile(outputPath, markdown);
  outro(pc2.green(`${pc2.bold("Done!")} Markdown saved to ${outputPath}`));
};

// src/cli/bin.ts
program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(package_default.version, "-v, --version", "output the current version");
program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
program.parse();