markitdown-js/dist/markitdown.js

Convert documents to markdown text content. Originally inspired by microsoft's markitdown python library.

// src/markitdown.ts
import path2 from "path";
import fs16 from "fs";
import axios from "axios";
import tmp2 from "tmp";
import Ffmpeg2 from "fluent-ffmpeg";

// src/converters/document.ts
var DocumentConverter = class _DocumentConverter {
  /**
   * Lower priority values are tried first.
   * Used for specific file formats like .docx, .pdf, .xlsx, or specific pages like Wikipedia.
   */
  static PRIORITY_SPECIFIC_FILE_FORMAT = 0;
  /**
   * Used for near catch-all converters for mimetypes like text/*, etc.
   * These are tried after more specific converters.
   */
  static PRIORITY_GENERIC_FILE_FORMAT = 10;
  /**
   * The priority of this converter.
   * @private
   */
  _priority;
  /**
   * Initialize the DocumentConverter with a given priority.
   *
   * Priorities work as follows: By default, most converters get priority
   * DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT (== 0). The exception
   * is the PlainTextConverter, which gets priority PRIORITY_GENERIC_FILE_FORMAT (== 10),
   * with lower values being tried first (i.e., higher priority).
   *
   * Just prior to conversion, the converters are sorted by priority, using
   * a stable sort. This means that converters with the same priority will
   * remain in the same order, with the most recently registered converters
   * appearing first.
   *
   * @param {number} priority - The priority of this converter
   */
  constructor(priority = _DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    this._priority = priority;
  }
  /**
   * Gets the priority of the converter in the converter list.
   * Lower values are tried first (higher priority).
   * @returns {number} The priority value
   */
  get priority() {
    return this._priority;
  }
  /**
   * Sets the priority of the converter.
   * @param {number} value - The new priority value
   */
  set priority(value) {
    this._priority = value;
  }
};

// src/converters/plainText.ts
import mime from "mime-types";
import fs from "fs";
import iconv from "iconv-lite";
var PlainTextConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_GENERIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a text file to the standard document format.
   * Automatically detects content type based on file extension and only processes
   * files that have text/* MIME types or application/json.
   *
   * @param {string} localPath - Path to the text file
   * @param {ConversionOptions} options - Conversion options including file extension
   * @returns {Promise<DocumentConverterResult>} Object containing the file content as textContent (title is null), or returns null if the file type is not supported
   * @throws {Error} If the file cannot be read or decoded
   */
  async convert(localPath, options) {
    const contentType = mime.lookup(
      `__placeholder${options.fileExtension || ""}`
    );
    if (!contentType) return null;
    const isValidType = contentType.toLowerCase().startsWith("text/") || contentType.toLowerCase() === "application/json";
    if (!isValidType) return null;
    const buffer = await fs.promises.readFile(localPath);
    const text = iconv.decode(buffer, "utf-8");
    return {
      title: null,
      textContent: text
    };
  }
};

// src/converters/html.ts
import { parse } from "node-html-parser";
import fs2 from "fs";

// src/converters/customMarkdown.ts
import TurndownService from "turndown";
var CustomMarkdownConverter = class {
  turndownService;
  /**
   * Initializes the Markdown converter with customized rules.
   * @param {TurndownOptions} [options={}] - Optional configuration settings for Turndown.
   */
  constructor(options = {}) {
    const defaultOptions = {
      headingStyle: "atx",
      hr: "---",
      bulletListMarker: "*",
      codeBlockStyle: "fenced",
      emDelimiter: "_",
      keepInlineImages: [],
      ...options
    };
    this.turndownService = new TurndownService(defaultOptions);
    this.turndownService.addRule("heading", {
      filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
      replacement: (content, node) => {
        const level = Number(node.nodeName.charAt(1));
        const prefix = "\n" + "#".repeat(level) + " ";
        return prefix + content.trim() + "\n";
      }
    });
    this.turndownService.addRule("link", {
      filter: "a",
      replacement: (content, node) => {
        const element = node;
        content = content.trim();
        if (!content) return "";
        let href = element.getAttribute("href");
        const title = element.getAttribute("title");
        if (href) {
          try {
            const url = new URL(href);
            if (!["http:", "https:", "file:"].includes(url.protocol)) {
              return content;
            }
            url.pathname = encodeURI(decodeURI(url.pathname));
            href = url.toString();
          } catch (error) {
            return content;
          }
        }
        if (content.replace(/\\_/g, "_") === href && !title) {
          return `<${href}>`;
        }
        const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : "";
        return href ? `[${content}](${href}${titlePart})` : content;
      }
    });
    this.turndownService.addRule("image", {
      filter: "img",
      replacement: (content, node) => {
        const element = node;
        const alt = element.getAttribute("alt") || "";
        let src = element.getAttribute("src") || "";
        const title = element.getAttribute("title") || "";
        const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : "";
        const keepInlineImages = this.turndownService.options.keepInlineImages;
        if (node.parentNode && (!keepInlineImages || !keepInlineImages.includes(node.parentNode.nodeName.toLowerCase()))) {
          return alt;
        }
        if (src.startsWith("data:")) {
          src = src.split(",")[0] + "...";
        }
        return `![${alt}](${src}${titlePart})`;
      }
    });
  }
  /**
   * Converts an HTML string into Markdown.
   * @param {string} html - The HTML content to convert.
   * @returns {string} The converted Markdown string.
   */
  convert(html) {
    return this.turndownService.turndown(html);
  }
  /**
   * Adds a custom Turndown rule.
   * @param {string} rule - The name of the custom rule.
   * @param {TurndownService.Rule} rules - The rule definition to be added.
   */
  addRule(rule, rules) {
    this.turndownService.addRule(rule, rules);
  }
};

// src/converters/html.ts
var HtmlConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_GENERIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts an HTML file to Markdown format.
   *
   * @param {string} localPath - Path to the local HTML file
   * @param {ConversionOptions} options - Conversion options
   * @param {string} [options.fileExtension] - File extension (must be .html or .htm)
   * @returns {Promise<DocumentConverterResult>} Conversion result or null if:
   *   - File extension is not .html or .htm
   *   - File cannot be read
   *   - Conversion fails
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (![".html", ".htm"].includes(extension)) {
      return null;
    }
    const content = await fs2.promises.readFile(localPath, "utf-8");
    return this._convert(content);
  }
  /**
   * Converts HTML content to Markdown format.
   * Internal method used by both direct HTML conversion and other converters.
   *
   * @param {string} htmlContent - Raw HTML content to convert
   * @returns {DocumentConverterResult} Object containing title and converted markdown content
   *
   * @remarks
   * - Removes all <script> and <style> elements before conversion
   * - Attempts to extract content from <body> first, falls back to entire document
   * - Preserves document title if available
   * - Trims whitespace from the final markdown
   *
   * @protected
   */
  _convert(htmlContent) {
    const root = parse(htmlContent);
    root.querySelectorAll("script, style").forEach((el) => el.remove());
    const markdownConverter = new CustomMarkdownConverter();
    const bodyElm = root.querySelector("body");
    const webpageText = bodyElm ? markdownConverter.convert(bodyElm.innerHTML) : markdownConverter.convert(root.innerHTML);
    return {
      title: root.querySelector("title")?.text || null,
      textContent: webpageText.trim()
    };
  }
};

// src/converters/rss.ts
import fs3 from "fs/promises";
import { parse as parse2 } from "node-html-parser";
import { DOMParser } from "xmldom";
var RSSConverter = class extends HtmlConverter {
  constructor() {
    super();
  }
  /**
   * Converts a feed file to markdown format.
   * Automatically detects the feed type (RSS, Atom, or XML) and processes accordingly.
   *
   * @param {string} localPath - Path to the feed file
   * @param {ConversionOptions} options - Conversion options including file extension
   * @returns {Promise<DocumentConverterResult>} Object containing formatted markdown with feed content, or returns null for unsupported file types or parsing failures
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "".toLowerCase();
    if (![".xml", ".rss", ".atom"].includes(extension)) {
      return null;
    }
    try {
      const content = await fs3.readFile(localPath, "utf-8");
      const parser = new DOMParser();
      const doc = parser.parseFromString(content, "text/xml");
      let result = null;
      if (doc.getElementsByTagName("rss").length) {
        result = this._parseRssType(doc);
      } else if (doc.getElementsByTagName("feed").length) {
        const root = doc.getElementsByTagName("feed")[0];
        if (root?.getElementsByTagName("entry").length) {
          result = this._parseAtomType(doc);
        }
      } else {
        result = this._parseXmlType(doc);
      }
      return result;
    } catch (error) {
      console.error("RSS parsing error:", error);
      return null;
    }
  }
  /**
   * Parses an Atom feed document into markdown.
   * Extracts feed metadata and entries, including titles, summaries, and content.
   *
   * @param {Document} doc - Parsed XML document
   * @returns {DocumentConverterResult} Formatted markdown content
   * @private
   */
  _parseAtomType(doc) {
    try {
      const root = doc.getElementsByTagName("feed")[0];
      const title = this._getDataByTagName(root, "title");
      const subtitle = this._getDataByTagName(root, "subtitle");
      const updated = this._getDataByTagName(root, "updated");
      const id = this._getDataByTagName(root, "id");
      const link = this._getDataByTagName(root, "link");
      const entries = root?.getElementsByTagName("entry") || [];
      let mdText = `# ${title}
`;
      if (subtitle) {
        mdText += `${subtitle}
`;
      }
      if (updated) {
        mdText += `Updated on: ${updated}
`;
      }
      if (id) {
        mdText += `ID: ${id}
`;
      }
      if (link) {
        mdText += `Link: ${link.getAttribute(
          "href"
        )}
`;
      }
      for (const entry of Array.from(entries)) {
        const entryTitle = this._getDataByTagName(
          entry,
          "title"
        );
        const entrySummary = this._getDataByTagName(
          entry,
          "summary"
        );
        const entryUpdated = this._getDataByTagName(
          entry,
          "updated"
        );
        const entryContent = this._getDataByTagName(
          entry,
          "content"
        );
        const entryId = this._getDataByTagName(entry, "id");
        const entryLink = this._getDataByTagName(entry, "link");
        if (entryTitle) mdText += `
## ${entryTitle}
`;
        if (entryUpdated) mdText += `Updated on: ${entryUpdated}
`;
        if (entrySummary) mdText += this._parseContent(entrySummary) + "\n";
        if (entryContent) mdText += this._parseContent(entryContent) + "\n";
        if (entryId) mdText += `ID: ${entryId}
`;
        if (entryLink)
          mdText += `Link: ${entryLink.getAttribute(
            "href"
          )}
`;
      }
      return {
        title,
        textContent: mdText
      };
    } catch (error) {
      console.error("Atom parsing error: ", error);
      return null;
    }
  }
  /**
   * Parses an RSS feed document into markdown.
   * Extracts channel metadata and items, including titles, descriptions, and content.
   *
   * @param {Document} doc - Parsed XML document
   * @returns {DocumentConverterResult} Formatted markdown content
   * @private
   */
  _parseRssType(doc) {
    try {
      const root = doc.getElementsByTagName("rss")[0];
      const channels = root?.getElementsByTagName("channel") || [];
      if (!channels.length) return null;
      const channel = channels[0];
      const channelTitle = this._getDataByTagName(
        channel,
        "title"
      );
      const channelDescription = this._getDataByTagName(
        channel,
        "description"
      );
      const channelLink = this._getDataByTagName(
        channel,
        "link"
      );
      const channelUpdated = this._getDataByTagName(
        channel,
        "lastBuildDate"
      );
      const items = channel.getElementsByTagName("item");
      let mdText = "";
      if (channelTitle) mdText += `# ${channelTitle}
`;
      if (channelUpdated) mdText += `Updated on: ${channelUpdated}
`;
      if (channelDescription) mdText += `${channelDescription}
`;
      if (channelLink) mdText += `ID: ${channelLink}
`;
      for (const item of Array.from(items)) {
        const title = this._getDataByTagName(item, "title");
        const description = this._getDataByTagName(
          item,
          "description"
        );
        const pubDate = this._getDataByTagName(item, "pubDate");
        const content = this._getDataByTagName(
          item,
          "content:encoded"
        );
        const link = this._getDataByTagName(item, "link");
        if (title) mdText += `
## ${title}.
`;
        if (pubDate) mdText += `Published on: ${pubDate}.
`;
        if (description) mdText += this._parseContent(description) + ".\n";
        if (content) mdText += this._parseContent(content) + ".\n";
        if (link) mdText += `ID: ${link}.
`;
        mdText += "\n";
      }
      return {
        title: channelTitle,
        textContent: mdText
      };
    } catch (error) {
      console.error("RSS parsing error:", error);
      return null;
    }
  }
  /**
   * Parses a generic XML document into markdown.
   * Creates a hierarchical markdown representation of the XML structure.
   *
   * @param {Document} doc - Parsed XML document
   * @returns {DocumentConverterResult} Formatted markdown content
   * @private
   */
  _parseXmlType(doc) {
    try {
      const items = doc.lastChild;
      let mdText = `# ${items.tagName}
`;
      mdText += this._parseXmlNode(items, 0);
      return {
        title: null,
        textContent: mdText
      };
    } catch (error) {
      console.error("XML parsing error:", error);
      return null;
    }
  }
  /**
   * Recursively parses XML nodes into a markdown list structure.
   *
   * @param {Element} items - XML element to parse
   * @param {number} tabCount - Current indentation level
   * @returns {string} Formatted markdown content
   * @private
   */
  _parseXmlNode(items, tabCount = 0) {
    let mdContent = "";
    let tabs = "  ".repeat(tabCount);
    const childNodes = Array.from(items.childNodes);
    childNodes.forEach((item) => {
      if (item.tagName) {
        const attributes = {};
        Object.entries(item.attributes || []).forEach(
          ([key, value]) => attributes[key] = ` ${value}`
        );
        if (attributes._ownerElement) delete attributes._ownerElement;
        if (attributes.length) delete attributes.length;
        const hasChildren = Array.from(item.childNodes || []).length > 1;
        mdContent += `
 ${tabs}- ${item.tagName}: ${!hasChildren ? item.textContent : ""}`;
        mdContent += Object.values(attributes).map((val) => "\n " + tabs + val.replace(/=/g, " - ")).join("");
        if (hasChildren) {
          mdContent += this._parseXmlNode(item, tabCount + 2);
        }
      }
    });
    return mdContent;
  }
  /**
   * Parses HTML content within feed entries.
   *
   * @param {string} content - HTML content to parse
   * @returns {string} Cleaned content
   * @private
   */
  _parseContent(content) {
    try {
      const root = parse2(content);
      return this._convert(root.innerHTML);
    } catch (error) {
      return content;
    }
  }
  /**
   * Extracts data from the first child element with the given tag name.
   *
   * @param {HTMLElement} element - Parent element to search
   * @param {string} tagName - Tag name to find
   * @returns {string | null} Content of the first matching element or null
   * @private
   */
  _getDataByTagName(element, tagName) {
    const nodes = element.getElementsByTagName(tagName);
    if (!nodes.length) return null;
    const firstChild = nodes[0]?.firstChild;
    return firstChild ? firstChild.nodeValue : null;
  }
};

// src/converters/wikipedia.ts
import { parse as parse3 } from "node-html-parser";
import fs4 from "fs";
var WikipediaConverter = class extends DocumentConverter {
  _turndown;
  /**
   * Initializes a new instance of WikipediaConverter and configures the markdown converter.
   */
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
    this._turndown = new CustomMarkdownConverter();
    this._configureTurndown();
  }
  /**
   * Configures the Turndown converter with Wikipedia-specific rules for handling
   * images and references.
   * @private
   */
  _configureTurndown() {
    this._turndown.addRule("image", {
      filter: ["img"],
      replacement: function(content, node) {
        const alt = node.getAttribute("alt") || "";
        const src = node.getAttribute("src") || "";
        const width = node.getAttribute("width");
        const height = node.getAttribute("height");
        let markdown = `![${alt}](${src.startsWith("//") ? "https:" + src : src})`;
        if (width && height) {
          markdown += `{: width="${width}" height="${height}"}`;
        }
        return markdown;
      }
    });
    this._turndown.addRule("wikiRef", {
      filter: (node) => {
        return node.classList && node.classList.contains("reference");
      },
      replacement: function(content, node) {
        return "";
      }
    });
  }
  /**
   * Converts a Wikipedia HTML file to Markdown format.
   * @param {string} localPath - The local file path to the Wikipedia HTML file
   * @param {ConversionOptions} options - Conversion options including file extension and URL
   * @returns {Promise<DocumentConverterResult>} The converted document or null if conversion fails
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (![".html", ".htm"].includes(extension.toLowerCase())) {
      return null;
    }
    const url = options.url || "";
    if (!/^https?:\/\/[a-zA-Z]{2,3}\.wikipedia\.org\//.test(url)) {
      return null;
    }
    let html;
    try {
      html = fs4.readFileSync(localPath, "utf-8");
    } catch (error) {
      console.error("Error reading file:", error);
      return null;
    }
    const root = parse3(html);
    const titleElm = root.querySelector("span.mw-page-title-main");
    const bodyElm = root.querySelector(
      "div#mw-content-text"
    );
    this.cleanupContent(bodyElm);
    const content = {
      title: titleElm?.textContent?.trim() ?? "",
      sections: this.parseSections(bodyElm)
    };
    return {
      title: content.title,
      textContent: this.generateMarkdown(
        content,
        root
      )
    };
  }
  /**
   * Removes unwanted elements from the Wikipedia content and cleans up the HTML structure.
   * @param {HTMLElement} element - The root element containing Wikipedia content
   * @private
   */
  cleanupContent(element) {
    const selectorsToRemove = [
      ".mw-editsection",
      ".reference",
      ".error",
      ".noprint",
      "#toc",
      ".toc",
      "style",
      "script",
      "table.infobox"
    ];
    selectorsToRemove.forEach((selector) => {
      element.querySelectorAll(selector).forEach((el) => {
        if (el.parentNode) {
          el.parentNode.removeChild(el);
        }
      });
    });
    const headings = element.querySelectorAll("h2");
    let seeAlsoHeading = null;
    for (const heading of Array.from(headings)) {
      if (heading.textContent?.trim() === "See also") {
        seeAlsoHeading = heading;
        break;
      }
    }
    if (seeAlsoHeading && seeAlsoHeading.parentNode) {
      const parentElement = seeAlsoHeading.parentNode;
      if (parentElement.parentNode) {
        let currentElement = parentElement;
        while (currentElement.nextSibling) {
          const nextElement = currentElement.nextSibling;
          if (nextElement.parentNode) {
            nextElement.parentNode.removeChild(nextElement);
          }
        }
        parentElement.parentNode.removeChild(parentElement);
      }
    }
  }
  /**
   * Creates an anchor ID from text by removing non-alphanumeric characters and converting to lowercase.
   * @param {string} text - The text to convert to an anchor ID
   * @returns {string} The formatted anchor ID
   * @private
   */
  _createAnchorId(text) {
    return text.toLowerCase().replace(/[^a-z0-9]+/g, "-");
  }
  /**
   * Parses the Wikipedia content into sections with titles and content.
   * @param {HTMLElement} mainContent - The main content element to parse
   * @returns {Section[]} An array of parsed sections
   * @private
   */
  parseSections(mainContent) {
    const sections = [];
    let currentSection = { title: "", content: "" };
    let inLeadSection = true;
    let currentHtml = "";
    mainContent.childNodes.forEach((element) => {
      const htmlElement = element;
      if (!htmlElement.tagName) return;
      if (htmlElement.tagName.match(/^H[1-6]$/i)) {
        if (currentHtml) {
          if (inLeadSection) {
            sections.push({
              title: "Introduction",
              level: 2,
              id: "introduction",
              content: this._turndown.convert(currentHtml)
            });
            inLeadSection = false;
          } else {
            const content = this._turndown.convert(currentHtml);
            sections.push({
              title: currentSection.title,
              content
            });
          }
          currentHtml = "";
        }
        const headingText = htmlElement.querySelector(".mw-headline")?.textContent || htmlElement.textContent?.trim();
        const level = htmlElement.tagName && htmlElement.tagName[1] ? parseInt(htmlElement.tagName[1]) : 5;
        const headingId = this._createAnchorId(headingText);
        currentSection = {
          title: headingText ?? "",
          level,
          id: headingId ?? "",
          content: ""
        };
      } else {
        currentHtml += htmlElement.outerHTML || "";
      }
    });
    if (currentHtml) {
      const content = this._turndown.convert(currentHtml);
      sections.push({
        ...currentSection,
        title: currentSection.title,
        content
      });
    }
    return sections;
  }
  /**
   * Generates a table of contents from the document headings.
   * @param {HTMLElement} root - The root element containing headings
   * @returns {string} Markdown formatted table of contents
   * @private
   */
  generateTableOfContents(root) {
    let toc = "## Table of Contents\n\n";
    root?.querySelectorAll(".mw-heading").forEach((el) => {
      const headingClassname = el.classList.toString();
      const headingLevel = headingClassname.endsWith("2") ? 1 : 2;
      const headingText = el?.textContent?.trim() ?? "";
      const headingId = this._createAnchorId(headingText);
      toc += `${"  ".repeat(headingLevel)} - [${headingText}](#${headingId.toLowerCase().replace(/\s+/g, "-")})
`;
    });
    return toc + "\n";
  }
  /**
   * Generates the final Markdown document from the parsed content.
   * @param {{title: string, sections: Section[]}} content - The parsed document content
   * @param {HTMLElement} root - The root HTML element
   * @returns {string} The complete Markdown document
   * @private
   */
  generateMarkdown(content, root) {
    let markdown = "";
    markdown += `# ${content.title}

`;
    markdown += this.generateTableOfContents(root);
    content.sections.forEach((section) => {
      if (section.title) {
        const headingLevel = "#".repeat(Math.min(section.level ?? 0, 6));
        markdown += `${headingLevel} [${section.title}](#${section.id})

`;
      }
      markdown += section.content + "\n\n";
    });
    return markdown;
  }
};

// src/converters/youtube.ts
import fs5 from "fs";
import { parse as parse4 } from "node-html-parser";
var YouTubeConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a YouTube video page to Markdown format.
   * Extracts video title, metadata, description, and transcript (if available).
   *
   * @param {string} localPath - The local file path to the YouTube page HTML file
   * @param {ConversionOptions} options - Conversion options including file extension and URL
   * @returns {Promise<DocumentConverterResult>} Object containing the converted markdown content, or null if the file is not a YouTube video page
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (![".html", ".htm"].includes(extension.toLowerCase())) {
      return null;
    }
    const url = options.url || "";
    if (!url.startsWith("https://www.youtube.com/watch?") && !url.startsWith("https://www.youtube.com/shorts/")) {
      return null;
    }
    const htmlContent = fs5.readFileSync(localPath, "utf-8");
    const root = parse4(htmlContent);
    const metadata = {
      title: root.querySelector("title")?.text || ""
    };
    root.querySelectorAll("meta").forEach((meta) => {
      const attr = meta.getAttribute("property") || meta.getAttribute("name") || meta.getAttribute("itemprop");
      if (attr && attr in metadata) {
        metadata[attr] = meta.getAttribute("content") || "";
      }
    });
    try {
      root.querySelectorAll("script").forEach((script) => {
        const content = script.text;
        if (content && content.includes("ytInitialData")) {
          const match = content.match(/var ytInitialData = ({.+?});/);
          if (match && match[1]) {
            try {
              const data = JSON.parse(match[1]);
              const attrdesc = this._findKey(
                data,
                "attributedDescriptionBodyText"
              );
              if (attrdesc) {
                metadata["description"] = attrdesc.content;
              }
            } catch (parseError) {
              console.debug("Failed to parse script content:", parseError);
            }
          }
        }
      });
    } catch (err) {
      console.error("Error parsing description:", err);
    }
    let webpageText = "# YouTube\n";
    const title = this._get(metadata, ["title", "og:title", "name"]) || "";
    if (title) {
      webpageText += `
## ${title}
`;
    }
    let stats = "";
    const views = this._get(metadata, ["interactionCount"]);
    if (views) stats += `- **Views:** ${views}
`;
    const keywords = this._get(metadata, ["keywords"]);
    if (keywords) stats += `- **Keywords:** ${keywords}
`;
    const runtime = this._get(metadata, ["duration"]);
    if (runtime) stats += `- **Runtime:** ${runtime}
`;
    if (stats) webpageText += `
### Video Metadata
${stats}
`;
    const description = this._get(metadata, ["description", "og:description"]);
    if (description) {
      webpageText += `
### Description
${description}
`;
    }
    let transcriptText = "";
    const parsedUrl = new URL(url);
    const videoId = parsedUrl.searchParams.get("v");
    const shortId = parsedUrl.pathname.split("/").pop();
    if (videoId || shortId) {
      try {
        await this.retryOperation(
          async () => {
            const response = await options.requestsSession?.get(
              `https://www.youtubetranscript.com/?server_vid2=${videoId || shortId}`
            );
            const transcript = parse4(response?.data);
            const transcriptTexts = [];
            transcript.querySelectorAll("transcript text").forEach((text) => {
              transcriptTexts.push(text.innerText);
            });
            transcriptText = transcriptTexts.join(" ");
          },
          3,
          2e3
        );
      } catch (err) {
        console.error("Error fetching transcript after retries:", err);
      }
    }
    if (transcriptText) {
      webpageText += `
### Transcript
${transcriptText}
`;
    }
    return {
      title,
      textContent: webpageText
    };
  }
  /**
   * Retries an operation with specified number of attempts and delay between retries
   *
   * @param {Function} operation - The async function to retry
   * @param {number} retries - Number of retry attempts
   * @param {number} delay - Delay between retries in milliseconds
   * @returns {Promise<any>} Result of the operation if successful
   * @throws {Error} If all retry attempts fail
   * @private
   */
  async retryOperation(operation, retries = 3, delay = 2e3) {
    let lastError;
    for (let attempt = 0; attempt < retries; attempt++) {
      try {
        return await operation();
      } catch (error) {
        const err = error;
        console.log(`Attempt ${attempt + 1} failed: ${err.message}`);
        lastError = err;
        if (attempt < retries - 1) {
          await new Promise((resolve) => setTimeout(resolve, delay));
        }
      }
    }
    throw new Error(
      `Operation failed after ${retries} attempts: ${lastError?.message}`
    );
  }
  /**
   * Retrieves a value from metadata using a list of possible keys.
   *
   * @param {Record<string, any>} metadata - The metadata object to search in
   * @param {string[]} keys - Array of possible keys to look for
   * @param {any} defaultValue - Value to return if no key is found
   * @returns {any} The first found value or the default value
   * @private
   */
  _get(metadata, keys, defaultValue = null) {
    for (const key of keys) {
      if (metadata[key]) {
        return metadata[key];
      }
    }
    return defaultValue;
  }
  /**
   * Recursively searches for a specific key in a nested object or array.
   *
   * @param {any} obj - The object or array to search in
   * @param {string} key - The key to search for
   * @returns {any} The value associated with the key if found, null otherwise
   * @private
   */
  _findKey(obj, key) {
    if (Array.isArray(obj)) {
      for (const item of obj) {
        const result = this._findKey(item, key);
        if (result) return result;
      }
    } else if (typeof obj === "object" && obj !== null) {
      for (const k in obj) {
        if (k === key) return obj[k];
        const result = this._findKey(obj[k], key);
        if (result) return result;
      }
    }
    return null;
  }
};

// src/converters/bingSerp.ts
import fs6 from "fs";
import { parse as parse5 } from "node-html-parser";
var BingSerpConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a Bing search results page to markdown format.
   * Only processes HTML files from Bing search URLs.
   *
   * @param {string} localPath - Path to the local HTML file
   * @param {ConversionOptions} options - Conversion options
   * @param {string} [options.fileExtension] - File extension (must be .html or .htm)
   * @param {string} [options.url] - Original URL (must be a Bing search URL)
   * @returns {Promise<DocumentConverterResult>} Conversion result containing search results
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (![".html", ".htm"].includes(extension.toLowerCase())) {
      return null;
    }
    const url = options.url || "";
    if (!/^https:\/\/www\.bing\.com\/search\?q=/.test(url)) {
      return null;
    }
    const parsedParams = new URL(url).searchParams;
    const query = parsedParams.get("q") || "";
    const html = fs6.readFileSync(localPath, "utf-8");
    const root = parse5(html);
    root.querySelectorAll(".tptt").forEach((el) => {
      el.textContent = el.textContent + " ";
    });
    root.querySelectorAll(".algoSlug_icon").forEach((el) => {
      el.remove();
    });
    let results = [];
    root.querySelectorAll(".b_algo").forEach((el) => {
      let resultText = el.textContent.trim();
      results.push(resultText);
    });
    let webpageText = `## A Bing search for '${query}' found the following results:

`;
    webpageText += results.join("\n\n");
    return {
      title: root.querySelector("title")?.innerText || null,
      textContent: webpageText
    };
  }
};

// src/converters/docx.ts
import fs7 from "fs";
import mammoth from "mammoth";
var DocxConverter = class extends HtmlConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a DOCX file to Markdown format.
   * Uses Mammoth.js to convert DOCX to HTML, then processes the HTML to Markdown.
   *
   * @param {string} localPath - Path to the local DOCX file
   * @param {ConversionOptions} options - Conversion options
   * @param {string} [options.fileExtension] - File extension (must be .docx)
   * @param {Array<string>} [options.styleMap] - Custom style mappings for Mammoth.js conversion
   * @returns {Promise<DocumentConverterResult>} Conversion result or null if:
   *   - File is not a DOCX
   *   - File cannot be read
   *   - Conversion fails
   *
   * @throws {Error} If file reading or conversion process fails
   * @override
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (extension.toLowerCase() !== ".docx") {
      return null;
    }
    try {
      const buffer = await fs7.promises.readFile(localPath);
      const styleMap = options.styleMap;
      const result = await mammoth.convertToHtml({ buffer }, { styleMap });
      const htmlContent = result.value;
      return this._convert(htmlContent);
    } catch (error) {
      console.error("Error converting DOCX file:", error);
      return null;
    }
  }
};

// src/converters/xlsx.ts
import XLSX from "xlsx";
var XlsxConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts an Excel file to Markdown format.
   * Each sheet is represented as a separate section with a Markdown table.
   *
   * @override
   * @param {string} localPath - The local file path to the Excel file
   * @param {ConversionOptions} options - Conversion options including file extension
   * @returns {Promise<DocumentConverterResult>} Object containing the converted markdown content, or returns null if the file is not an Excel file
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (![".xlsx", ".xls"].includes(extension.toLowerCase())) {
      return null;
    }
    const workbook = XLSX.readFile(localPath, { bookVBA: true });
    let mdContent = "";
    workbook.SheetNames.forEach((sheetName) => {
      const worksheet = workbook.Sheets[sheetName];
      if (!worksheet) return;
      mdContent += `## ${sheetName}
`;
      const tableContent = XLSX.utils.sheet_to_json(worksheet, {});
      const firstRow = tableContent[0];
      const tableHead = Object.keys(firstRow);
      const tableBody = tableContent.map((row) => {
        const tableRow = Object.values(row).map((cell) => `${cell}`).join(" | ");
        return tableRow;
      });
      const mdTable = [
        // Header row with column names
        `|${tableHead.join("|")}`,
        // Separator row with dashes
        `${tableHead.map((key) => "-".repeat(key.length)).join("|")}`,
        // Data rows
        `${tableBody.map((row) => `|${row}|`).join("\n")}`
      ].join("\n");
      mdContent += mdTable + "\n\n";
    });
    return {
      title: null,
      textContent: mdContent.trim()
    };
  }
};

// src/converters/pptx.ts
import nodePptxParser from "node-pptx-parser";
var PptxParser = "default" in nodePptxParser ? nodePptxParser.default : nodePptxParser;
var PptxConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a PPTX file to markdown format.
   * Extracts text from each slide and organizes them in order.
   * Slides are sorted by their ID to maintain presentation order.
   *
   * @param {string} localPath - Path to the PPTX file
   * @param {ConversionOptions} options - Conversion options including file extension
   * @returns {Promise<DocumentConverterResult>} Object containing formatted markdown as textContent (title is null), or returns null for unsupported file types (.ppt files or non-PowerPoint files)
   * @throws {Error} If the file cannot be read or parsed
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (extension.toLowerCase() === ".ppt") {
      console.warn("PPT files are not supported. Please use PPTX files.");
      return null;
    }
    if (extension.toLowerCase() !== ".pptx") {
      return null;
    }
    const pptxParser = new PptxParser(localPath);
    const pptxSlides = await pptxParser.extractText();
    const sortedSlides = pptxSlides.sort((a, b) => {
      const aId = a.id.substring(3);
      const bId = b.id.substring(3);
      return Number(aId) - Number(bId);
    });
    let mdContent = "";
    sortedSlides.forEach((slide, index) => {
      const slideName = slide.path?.split("/").pop()?.split(".")[0] || `Slide ${index}`;
      mdContent += `
## ${slideName} - ${slide.id}
`;
      mdContent += slide.text ? slide.text : "Empty Slide...";
    });
    return {
      title: null,
      textContent: mdContent.replace(/\n\n\n/g, "\n").trim()
    };
  }
};

// src/converters/audio.ts
import fs8 from "fs";

// src/converters/media.ts
import { exiftool } from "exiftool-vendored";
var MediaConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a local file to the target format.
   * @abstract
   * @param {string} localPath - The path to the local file to convert
   * @param {ConversionOptions} [options] - Optional conversion configuration
   * @returns {Promise<DocumentConverterResult>} A promise that resolves with the conversion result
   * @throws {Error} May throw implementation-specific errors during conversion
   */
  convert(localPath, options) {
    throw new Error("Method not implemented.");
  }
  /**
   * Generates media metadata using exiftool.
   * @param {string} localPath - Path to the local audio file
   * @returns {Promise<Record<string, any> | null>} The transcription text or null if transcription fails or is unavailable
   */
  async _getMetadata(localPath) {
    try {
      const metadata = await exiftool.read(localPath);
      return metadata;
    } catch (error) {
      console.error("Error reading metadata:", error);
      return null;
    } finally {
      exiftool.end();
    }
  }
};

// src/converters/audio.ts
var AudioConverter = class extends MediaConverter {
  /**
   * Converts an audio file to markdown format, including metadata and transcription.
   * @param {string} localPath - Path to the local audio file
   * @param {ConversionOptions} options - Conversion options including file extension and LLM callback
   * @param {string} options.fileExtension - The file extension of the audio file
   * @param {LlmCall} [options.llmCall] - Callback function for audio transcription
   * @returns {Promise<DocumentConverterResult>} The conversion result or null if file type not supported
   * @override
   */
  async convert(localPath, options) {
    const supportedAudioExtensions = [".m4a", ".mp3", ".mpga", ".wav"];
    if (!supportedAudioExtensions.includes(options.fileExtension.toLowerCase()))
      return null;
    let mdContent = "";
    const metadata = await this._getMetadata(localPath);
    if (metadata) {
      [
        "Title",
        "Artist",
        "Author",
        "Band",
        "Album",
        "Genre",
        "Track",
        "DateTimeOriginal",
        "CreateDate",
        "Duration"
      ].forEach((field) => {
        if (metadata[field]) {
          mdContent += `- **${field}**: ${metadata[field]}
`;
        }
      });
    }
    try {
      const transcript = await this._transcribeAudio(
        localPath,
        options.llmCall
      );
      mdContent += `
# Audio Transcript:

${transcript || "[No speech detected]"}`;
    } catch (err) {
      console.error(err);
      mdContent += "\n## Audio Transcript:\nError. Could not transcribe this audio.";
    }
    return {
      title: null,
      textContent: mdContent.trim()
    };
  }
  /**
   * Transcribes audio content using the provided LLM callback function.
   * @param {string} localPath - Path to the local audio file
   * @param {LlmCall} llmCall - Callback function for audio transcription
   * @returns {Promise<string | null>} The transcription text or null if transcription fails or is unavailable
   */
  async _transcribeAudio(localPath, llmCall) {
    if (typeof llmCall !== "function") return null;
    try {
      const file = fs8.createReadStream(localPath);
      const response = await llmCall({ file });
      return response;
    } catch (error) {
      console.error("Error in transcription:", error);
      return null;
    }
  }
};

// src/converters/video.ts
import fs9 from "fs";
import tmp from "tmp";
import Ffmpeg from "fluent-ffmpeg";
var VideoConverter = class extends AudioConverter {
  /**
   * Converts a video file to markdown format.
   * Extracts available metadata and optionally transcribes audio content if FFmpeg is available.
   *
   * @param {string} localPath - Path to the video file
   * @param {ConversionOptions} options - Conversion options including:
   *   - fileExtension: The file extension (must be .mp4, .mkv, .webm, or .mpeg)
   *   - llmCall: Optional function for audio transcription
   * @returns {Promise<DocumentConverterResult>} Object containing extracted metadata and optional transcript,or returns null for unsupported file types
   * @throws {Error} If file processing or transcription fails
   */
  async convert(localPath, options) {
    const supportedVideoExtensions = [".mp4", ".mkv", ".webm", ".mpeg"];
    if (!supportedVideoExtensions.includes(options.fileExtension)) return null;
    let mdContent = "";
    const metadata = await this._getMetadata(localPath);
    if (metadata) {
      [
        "Title",
        "Artist",
        "Author",
        "Band",
        "Album",
        "Genre",
        "Track",
        "DateTimeOriginal",
        "CreateDate",
        "Duration"
      ].forEach((field) => {
        if (metadata[field]) {
          mdContent += `- **${field}**: ${metadata[field]}
`;
        }
      });
    }
    if (typeof options.llmCall === "function" && global.IS_FFMPEG_CAPABLE) {
      const tempPath = tmp.tmpNameSync({
        prefix: "markitdownjs-",
        postfix: ".wav"
      });
      try {
        await new Promise((resolve, reject) => {
          Ffmpeg(localPath).toFormat("wav").save(tempPath).on("end", resolve).on("error", reject);
        }).catch((err) => {
          console.error("Error in video conversion:", err);
        });
        const transcript = await super._transcribeAudio(
          localPath,
          options.llmCall
        );
        mdContent += `

### Audio Transcript:
${transcript || "[No speech detected]"}`;
      } catch (err) {
        mdContent += "\n\n### Audio Transcript:\nError. Could not transcribe this audio.";
      } finally {
        fs9.unlinkSync(tempPath);
      }
    }
    return { title: null, textContent: mdContent.trim() };
  }
};

// src/converters/image.ts
import fs10 from "fs";
import mime2 from "mime-types";
import tesseract from "node-tesseract-ocr";
var ImageConverter = class extends MediaConverter {
  /**
   * Converts an image file to markdown format with metadata, OCR text, and AI description.
   *
   * @param {string} localPath - Path to the local image file
   * @param {ConversionOptions} options - Conversion options
   * @param {string} [options.fileExtension] - File extension (must be .jpg, .jpeg, or .png)
   * @param {LlmCall} [options.llmCall] - Callback function for LLM image description
   * @returns {Promise<DocumentConverterResult>} Conversion result or null if:
   *   - File is not a supported image type
   *   - File cannot be read
   *
   * @remarks
   * The converter attempts to extract three types of information:
   * 1. Metadata fields: ImageSize, Title, Caption, Description, Keywords, Artist,
   *    Author, DateTimeOriginal, CreateDate, GPSPosition
   * 2. OCR text (requires Tesseract installation)
   * 3. AI-generated description (requires configured llmCall)
   * @override
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (![".jpg", ".jpeg", ".png"].includes(extension.toLowerCase())) {
      return null;
    }
    let mdContent = "";
    const metadata = await this._getMetadata(localPath);
    if (metadata) {
      const fields = [
        "ImageSize",
        "Title",
        "Caption",
        "Description",
        "Keywords",
        "Artist",
        "Author",
        "DateTimeOriginal",
        "CreateDate",
        "GPSPosition"
      ];
      fields.forEach((field) => {
        if (metadata[field]) {
          mdContent += `- **${field}**: ${metadata[field]}
`;
        }
      });
    }
    try {
      const text = await tesseract.recognize(localPath);
      if (text) mdContent += `
# Text:

${text.trim()}`;
    } catch (error) {
    }
    const description = await this._getLlmDescription({
      localPath,
      fileExtension: extension,
      llmCall: options.llmCall
    });
    if (description) mdContent += `

# Description:

${description.trim()}`;
    return {
      title: null,
      textContent: mdContent
    };
  }
  /**
   * Gets an AI-generated description of the image using the provided LLM callback.
   *
   * @param {Object} params - Parameters for LLM description generation
   * @param {string} params.localPath - Path to the image file
   * @param {string} params.fileExtension - File extension for MIME type determination
   * @param {LlmCall} params.llmCall - Callback function for LLM processing
   * @returns {Promise<string | null>} Generated description or null if:
   *   - LLM callback is not provided
   *   - Image processing fails
   *   - LLM call fails
   *
   * @private
   */
  async _getLlmDescription({
    localPath,
    fileExtension,
    llmCall
  }) {
    if (typeof llmCall !== "function") return null;
    try {
      const imageBase64 = fs10.readFileSync(localPath, { encoding: "base64" });
      const contentType = mime2.lookup(fileExtension) || "image/jpeg";
      const dataUri = `data:${contentType};base64,${imageBase64}`;
      const messages = [
        {
          role: "user",
          content: [
            {
              type: "text",
              text: "Write a detailed caption for this base64 image string."
            },
            { type: "image_url", image_url: { url: dataUri } }
          ]
        }
      ];
      const response = await llmCall({
        messages,
        imageBase64
      });
      return response;
    } catch (err) {
      console.error("error making llmCall: ", err);
      return null;
    }
  }
};

// src/converters/ipynb.ts
import fs11 from "fs";
var IpynbConverter = class extends DocumentConverter {
  constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
    super(priority);
  }
  /**
   * Converts a Jupyter Notebook file to markdown format.
   *
   * @param {string} localPath - The local file system path to the .ipynb file
   * @param {ConversionOptions} options - Conversion options including file extension
   * @returns {Promise<DocumentConverterResult>} A promise that resolves to the conversion result
   * @throws {Error} If the file cannot be read or parsed
   */
  async convert(localPath, options) {
    const extension = options.fileExtension || "";
    if (extension.toLowerCase() !== ".ipynb") {
      return null;
    }
    const notebookContent = JSON.parse(fs11.readFileSync(localPath, "utf-8"));
    return this._convert(notebookContent);
  }
  /**
   * Internal method to convert parsed notebook content to markdown format.
   * Processes both markdown and code cells, including their outputs.
   *
   * @param {JupyterNotebook} notebookContent - Parsed Jupyter notebook content
   * @returns {DocumentConverterResult} Converted document with title and markdown content
   * @throws {Error} If conversion process fails
   * @private
   */
  _convert(notebookContent) {
    try {
      let mdOutput = [];
      let title = null;
      notebookContent.cells.forEach((cell) => {
        if (cell.cell_type === "markdown") {
          const markdownText = cell.source.join("");
          if (!title) {
            for (const line of cell.source) {
              if (line.startsWith("# ")) {
                title = line.replace(/^# /, "").trim();
                break;
              }
            }
          }
          mdOutput.push(markdownText);
        } else if (cell.cell_type === "code") {
          mdOutput.push("```python\n" + cell.source.join("") + "\n```");
          if (cell.outputs && cell.o