markitdown-js
Version:
Convert documents to markdown text content. Originally inspired by microsoft's markitdown python library.
1,436 lines (1,412 loc) • 74.7 kB
JavaScript
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/markitdown.ts
var markitdown_exports = {};
__export(markitdown_exports, {
AudioConverter: () => AudioConverter,
BingSerpConverter: () => BingSerpConverter,
CustomMarkdownConverter: () => CustomMarkdownConverter,
DocumentConverter: () => DocumentConverter,
DocumentIntelligenceConverter: () => DocumentIntelligenceConverter,
DocxConverter: () => DocxConverter,
HtmlConverter: () => HtmlConverter,
ImageConverter: () => ImageConverter,
IpynbConverter: () => IpynbConverter,
MarkItDown: () => MarkItDown,
MediaConverter: () => MediaConverter,
OutlookMsgConverter: () => OutlookMsgConverter,
PdfConverter: () => PdfConverter,
PlainTextConverter: () => PlainTextConverter,
PptxConverter: () => PptxConverter,
RSSConverter: () => RSSConverter,
VideoConverter: () => VideoConverter,
WikipediaConverter: () => WikipediaConverter,
XlsxConverter: () => XlsxConverter,
YouTubeConverter: () => YouTubeConverter,
ZipConverter: () => ZipConverter,
default: () => MarkItDown
});
module.exports = __toCommonJS(markitdown_exports);
var import_path2 = __toESM(require("path"), 1);
var import_fs15 = __toESM(require("fs"), 1);
var import_axios = __toESM(require("axios"), 1);
var import_tmp2 = __toESM(require("tmp"), 1);
var import_fluent_ffmpeg2 = __toESM(require("fluent-ffmpeg"), 1);
// src/converters/document.ts
var DocumentConverter = class _DocumentConverter {
/**
* Lower priority values are tried first.
* Used for specific file formats like .docx, .pdf, .xlsx, or specific pages like Wikipedia.
*/
static PRIORITY_SPECIFIC_FILE_FORMAT = 0;
/**
* Used for near catch-all converters for mimetypes like text/*, etc.
* These are tried after more specific converters.
*/
static PRIORITY_GENERIC_FILE_FORMAT = 10;
/**
* The priority of this converter.
* @private
*/
_priority;
/**
* Initialize the DocumentConverter with a given priority.
*
* Priorities work as follows: By default, most converters get priority
* DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT (== 0). The exception
* is the PlainTextConverter, which gets priority PRIORITY_GENERIC_FILE_FORMAT (== 10),
* with lower values being tried first (i.e., higher priority).
*
* Just prior to conversion, the converters are sorted by priority, using
* a stable sort. This means that converters with the same priority will
* remain in the same order, with the most recently registered converters
* appearing first.
*
* @param {number} priority - The priority of this converter
*/
constructor(priority = _DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
this._priority = priority;
}
/**
* Gets the priority of the converter in the converter list.
* Lower values are tried first (higher priority).
* @returns {number} The priority value
*/
get priority() {
return this._priority;
}
/**
* Sets the priority of the converter.
* @param {number} value - The new priority value
*/
set priority(value) {
this._priority = value;
}
};
// src/converters/plainText.ts
var import_mime_types = __toESM(require("mime-types"), 1);
var import_fs = __toESM(require("fs"), 1);
var import_iconv_lite = __toESM(require("iconv-lite"), 1);
var PlainTextConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_GENERIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts a text file to the standard document format.
* Automatically detects content type based on file extension and only processes
* files that have text/* MIME types or application/json.
*
* @param {string} localPath - Path to the text file
* @param {ConversionOptions} options - Conversion options including file extension
* @returns {Promise<DocumentConverterResult>} Object containing the file content as textContent (title is null), or returns null if the file type is not supported
* @throws {Error} If the file cannot be read or decoded
*/
async convert(localPath, options) {
const contentType = import_mime_types.default.lookup(
`__placeholder${options.fileExtension || ""}`
);
if (!contentType) return null;
const isValidType = contentType.toLowerCase().startsWith("text/") || contentType.toLowerCase() === "application/json";
if (!isValidType) return null;
const buffer = await import_fs.default.promises.readFile(localPath);
const text = import_iconv_lite.default.decode(buffer, "utf-8");
return {
title: null,
textContent: text
};
}
};
// src/converters/html.ts
var import_node_html_parser = require("node-html-parser");
var import_fs2 = __toESM(require("fs"), 1);
// src/converters/customMarkdown.ts
var import_turndown = __toESM(require("turndown"), 1);
var CustomMarkdownConverter = class {
turndownService;
/**
* Initializes the Markdown converter with customized rules.
* @param {TurndownOptions} [options={}] - Optional configuration settings for Turndown.
*/
constructor(options = {}) {
const defaultOptions = {
headingStyle: "atx",
hr: "---",
bulletListMarker: "*",
codeBlockStyle: "fenced",
emDelimiter: "_",
keepInlineImages: [],
...options
};
this.turndownService = new import_turndown.default(defaultOptions);
this.turndownService.addRule("heading", {
filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
replacement: (content, node) => {
const level = Number(node.nodeName.charAt(1));
const prefix = "\n" + "#".repeat(level) + " ";
return prefix + content.trim() + "\n";
}
});
this.turndownService.addRule("link", {
filter: "a",
replacement: (content, node) => {
const element = node;
content = content.trim();
if (!content) return "";
let href = element.getAttribute("href");
const title = element.getAttribute("title");
if (href) {
try {
const url = new URL(href);
if (!["http:", "https:", "file:"].includes(url.protocol)) {
return content;
}
url.pathname = encodeURI(decodeURI(url.pathname));
href = url.toString();
} catch (error) {
return content;
}
}
if (content.replace(/\\_/g, "_") === href && !title) {
return `<${href}>`;
}
const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : "";
return href ? `[${content}](${href}${titlePart})` : content;
}
});
this.turndownService.addRule("image", {
filter: "img",
replacement: (content, node) => {
const element = node;
const alt = element.getAttribute("alt") || "";
let src = element.getAttribute("src") || "";
const title = element.getAttribute("title") || "";
const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : "";
const keepInlineImages = this.turndownService.options.keepInlineImages;
if (node.parentNode && (!keepInlineImages || !keepInlineImages.includes(node.parentNode.nodeName.toLowerCase()))) {
return alt;
}
if (src.startsWith("data:")) {
src = src.split(",")[0] + "...";
}
return ``;
}
});
}
/**
* Converts an HTML string into Markdown.
* @param {string} html - The HTML content to convert.
* @returns {string} The converted Markdown string.
*/
convert(html) {
return this.turndownService.turndown(html);
}
/**
* Adds a custom Turndown rule.
* @param {string} rule - The name of the custom rule.
* @param {TurndownService.Rule} rules - The rule definition to be added.
*/
addRule(rule, rules) {
this.turndownService.addRule(rule, rules);
}
};
// src/converters/html.ts
var HtmlConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_GENERIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts an HTML file to Markdown format.
*
* @param {string} localPath - Path to the local HTML file
* @param {ConversionOptions} options - Conversion options
* @param {string} [options.fileExtension] - File extension (must be .html or .htm)
* @returns {Promise<DocumentConverterResult>} Conversion result or null if:
* - File extension is not .html or .htm
* - File cannot be read
* - Conversion fails
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (![".html", ".htm"].includes(extension)) {
return null;
}
const content = await import_fs2.default.promises.readFile(localPath, "utf-8");
return this._convert(content);
}
/**
* Converts HTML content to Markdown format.
* Internal method used by both direct HTML conversion and other converters.
*
* @param {string} htmlContent - Raw HTML content to convert
* @returns {DocumentConverterResult} Object containing title and converted markdown content
*
* @remarks
* - Removes all <script> and <style> elements before conversion
* - Attempts to extract content from <body> first, falls back to entire document
* - Preserves document title if available
* - Trims whitespace from the final markdown
*
* @protected
*/
_convert(htmlContent) {
const root = (0, import_node_html_parser.parse)(htmlContent);
root.querySelectorAll("script, style").forEach((el) => el.remove());
const markdownConverter = new CustomMarkdownConverter();
const bodyElm = root.querySelector("body");
const webpageText = bodyElm ? markdownConverter.convert(bodyElm.innerHTML) : markdownConverter.convert(root.innerHTML);
return {
title: root.querySelector("title")?.text || null,
textContent: webpageText.trim()
};
}
};
// src/converters/rss.ts
var import_promises = __toESM(require("fs/promises"), 1);
var import_node_html_parser2 = require("node-html-parser");
var import_xmldom = require("xmldom");
var RSSConverter = class extends HtmlConverter {
constructor() {
super();
}
/**
* Converts a feed file to markdown format.
* Automatically detects the feed type (RSS, Atom, or XML) and processes accordingly.
*
* @param {string} localPath - Path to the feed file
* @param {ConversionOptions} options - Conversion options including file extension
* @returns {Promise<DocumentConverterResult>} Object containing formatted markdown with feed content, or returns null for unsupported file types or parsing failures
*/
async convert(localPath, options) {
const extension = options.fileExtension || "".toLowerCase();
if (![".xml", ".rss", ".atom"].includes(extension)) {
return null;
}
try {
const content = await import_promises.default.readFile(localPath, "utf-8");
const parser = new import_xmldom.DOMParser();
const doc = parser.parseFromString(content, "text/xml");
let result = null;
if (doc.getElementsByTagName("rss").length) {
result = this._parseRssType(doc);
} else if (doc.getElementsByTagName("feed").length) {
const root = doc.getElementsByTagName("feed")[0];
if (root?.getElementsByTagName("entry").length) {
result = this._parseAtomType(doc);
}
} else {
result = this._parseXmlType(doc);
}
return result;
} catch (error) {
console.error("RSS parsing error:", error);
return null;
}
}
/**
* Parses an Atom feed document into markdown.
* Extracts feed metadata and entries, including titles, summaries, and content.
*
* @param {Document} doc - Parsed XML document
* @returns {DocumentConverterResult} Formatted markdown content
* @private
*/
_parseAtomType(doc) {
try {
const root = doc.getElementsByTagName("feed")[0];
const title = this._getDataByTagName(root, "title");
const subtitle = this._getDataByTagName(root, "subtitle");
const updated = this._getDataByTagName(root, "updated");
const id = this._getDataByTagName(root, "id");
const link = this._getDataByTagName(root, "link");
const entries = root?.getElementsByTagName("entry") || [];
let mdText = `# ${title}
`;
if (subtitle) {
mdText += `${subtitle}
`;
}
if (updated) {
mdText += `Updated on: ${updated}
`;
}
if (id) {
mdText += `ID: ${id}
`;
}
if (link) {
mdText += `Link: ${link.getAttribute(
"href"
)}
`;
}
for (const entry of Array.from(entries)) {
const entryTitle = this._getDataByTagName(
entry,
"title"
);
const entrySummary = this._getDataByTagName(
entry,
"summary"
);
const entryUpdated = this._getDataByTagName(
entry,
"updated"
);
const entryContent = this._getDataByTagName(
entry,
"content"
);
const entryId = this._getDataByTagName(entry, "id");
const entryLink = this._getDataByTagName(entry, "link");
if (entryTitle) mdText += `
## ${entryTitle}
`;
if (entryUpdated) mdText += `Updated on: ${entryUpdated}
`;
if (entrySummary) mdText += this._parseContent(entrySummary) + "\n";
if (entryContent) mdText += this._parseContent(entryContent) + "\n";
if (entryId) mdText += `ID: ${entryId}
`;
if (entryLink)
mdText += `Link: ${entryLink.getAttribute(
"href"
)}
`;
}
return {
title,
textContent: mdText
};
} catch (error) {
console.error("Atom parsing error: ", error);
return null;
}
}
/**
* Parses an RSS feed document into markdown.
* Extracts channel metadata and items, including titles, descriptions, and content.
*
* @param {Document} doc - Parsed XML document
* @returns {DocumentConverterResult} Formatted markdown content
* @private
*/
_parseRssType(doc) {
try {
const root = doc.getElementsByTagName("rss")[0];
const channels = root?.getElementsByTagName("channel") || [];
if (!channels.length) return null;
const channel = channels[0];
const channelTitle = this._getDataByTagName(
channel,
"title"
);
const channelDescription = this._getDataByTagName(
channel,
"description"
);
const channelLink = this._getDataByTagName(
channel,
"link"
);
const channelUpdated = this._getDataByTagName(
channel,
"lastBuildDate"
);
const items = channel.getElementsByTagName("item");
let mdText = "";
if (channelTitle) mdText += `# ${channelTitle}
`;
if (channelUpdated) mdText += `Updated on: ${channelUpdated}
`;
if (channelDescription) mdText += `${channelDescription}
`;
if (channelLink) mdText += `ID: ${channelLink}
`;
for (const item of Array.from(items)) {
const title = this._getDataByTagName(item, "title");
const description = this._getDataByTagName(
item,
"description"
);
const pubDate = this._getDataByTagName(item, "pubDate");
const content = this._getDataByTagName(
item,
"content:encoded"
);
const link = this._getDataByTagName(item, "link");
if (title) mdText += `
## ${title}.
`;
if (pubDate) mdText += `Published on: ${pubDate}.
`;
if (description) mdText += this._parseContent(description) + ".\n";
if (content) mdText += this._parseContent(content) + ".\n";
if (link) mdText += `ID: ${link}.
`;
mdText += "\n";
}
return {
title: channelTitle,
textContent: mdText
};
} catch (error) {
console.error("RSS parsing error:", error);
return null;
}
}
/**
* Parses a generic XML document into markdown.
* Creates a hierarchical markdown representation of the XML structure.
*
* @param {Document} doc - Parsed XML document
* @returns {DocumentConverterResult} Formatted markdown content
* @private
*/
_parseXmlType(doc) {
try {
const items = doc.lastChild;
let mdText = `# ${items.tagName}
`;
mdText += this._parseXmlNode(items, 0);
return {
title: null,
textContent: mdText
};
} catch (error) {
console.error("XML parsing error:", error);
return null;
}
}
/**
* Recursively parses XML nodes into a markdown list structure.
*
* @param {Element} items - XML element to parse
* @param {number} tabCount - Current indentation level
* @returns {string} Formatted markdown content
* @private
*/
_parseXmlNode(items, tabCount = 0) {
let mdContent = "";
let tabs = " ".repeat(tabCount);
const childNodes = Array.from(items.childNodes);
childNodes.forEach((item) => {
if (item.tagName) {
const attributes = {};
Object.entries(item.attributes || []).forEach(
([key, value]) => attributes[key] = ` ${value}`
);
if (attributes._ownerElement) delete attributes._ownerElement;
if (attributes.length) delete attributes.length;
const hasChildren = Array.from(item.childNodes || []).length > 1;
mdContent += `
${tabs}- ${item.tagName}: ${!hasChildren ? item.textContent : ""}`;
mdContent += Object.values(attributes).map((val) => "\n " + tabs + val.replace(/=/g, " - ")).join("");
if (hasChildren) {
mdContent += this._parseXmlNode(item, tabCount + 2);
}
}
});
return mdContent;
}
/**
* Parses HTML content within feed entries.
*
* @param {string} content - HTML content to parse
* @returns {string} Cleaned content
* @private
*/
_parseContent(content) {
try {
const root = (0, import_node_html_parser2.parse)(content);
return this._convert(root.innerHTML);
} catch (error) {
return content;
}
}
/**
* Extracts data from the first child element with the given tag name.
*
* @param {HTMLElement} element - Parent element to search
* @param {string} tagName - Tag name to find
* @returns {string | null} Content of the first matching element or null
* @private
*/
_getDataByTagName(element, tagName) {
const nodes = element.getElementsByTagName(tagName);
if (!nodes.length) return null;
const firstChild = nodes[0]?.firstChild;
return firstChild ? firstChild.nodeValue : null;
}
};
// src/converters/wikipedia.ts
var import_node_html_parser3 = require("node-html-parser");
var import_fs3 = __toESM(require("fs"), 1);
var WikipediaConverter = class extends DocumentConverter {
_turndown;
/**
* Initializes a new instance of WikipediaConverter and configures the markdown converter.
*/
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
this._turndown = new CustomMarkdownConverter();
this._configureTurndown();
}
/**
* Configures the Turndown converter with Wikipedia-specific rules for handling
* images and references.
* @private
*/
_configureTurndown() {
this._turndown.addRule("image", {
filter: ["img"],
replacement: function(content, node) {
const alt = node.getAttribute("alt") || "";
const src = node.getAttribute("src") || "";
const width = node.getAttribute("width");
const height = node.getAttribute("height");
let markdown = `![${alt}](${src.startsWith("//") ? "https:" + src : src})`;
if (width && height) {
markdown += `{: width="${width}" height="${height}"}`;
}
return markdown;
}
});
this._turndown.addRule("wikiRef", {
filter: (node) => {
return node.classList && node.classList.contains("reference");
},
replacement: function(content, node) {
return "";
}
});
}
/**
* Converts a Wikipedia HTML file to Markdown format.
* @param {string} localPath - The local file path to the Wikipedia HTML file
* @param {ConversionOptions} options - Conversion options including file extension and URL
* @returns {Promise<DocumentConverterResult>} The converted document or null if conversion fails
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (![".html", ".htm"].includes(extension.toLowerCase())) {
return null;
}
const url = options.url || "";
if (!/^https?:\/\/[a-zA-Z]{2,3}\.wikipedia\.org\//.test(url)) {
return null;
}
let html;
try {
html = import_fs3.default.readFileSync(localPath, "utf-8");
} catch (error) {
console.error("Error reading file:", error);
return null;
}
const root = (0, import_node_html_parser3.parse)(html);
const titleElm = root.querySelector("span.mw-page-title-main");
const bodyElm = root.querySelector(
"div#mw-content-text"
);
this.cleanupContent(bodyElm);
const content = {
title: titleElm?.textContent?.trim() ?? "",
sections: this.parseSections(bodyElm)
};
return {
title: content.title,
textContent: this.generateMarkdown(
content,
root
)
};
}
/**
* Removes unwanted elements from the Wikipedia content and cleans up the HTML structure.
* @param {HTMLElement} element - The root element containing Wikipedia content
* @private
*/
cleanupContent(element) {
const selectorsToRemove = [
".mw-editsection",
".reference",
".error",
".noprint",
"#toc",
".toc",
"style",
"script",
"table.infobox"
];
selectorsToRemove.forEach((selector) => {
element.querySelectorAll(selector).forEach((el) => {
if (el.parentNode) {
el.parentNode.removeChild(el);
}
});
});
const headings = element.querySelectorAll("h2");
let seeAlsoHeading = null;
for (const heading of Array.from(headings)) {
if (heading.textContent?.trim() === "See also") {
seeAlsoHeading = heading;
break;
}
}
if (seeAlsoHeading && seeAlsoHeading.parentNode) {
const parentElement = seeAlsoHeading.parentNode;
if (parentElement.parentNode) {
let currentElement = parentElement;
while (currentElement.nextSibling) {
const nextElement = currentElement.nextSibling;
if (nextElement.parentNode) {
nextElement.parentNode.removeChild(nextElement);
}
}
parentElement.parentNode.removeChild(parentElement);
}
}
}
/**
* Creates an anchor ID from text by removing non-alphanumeric characters and converting to lowercase.
* @param {string} text - The text to convert to an anchor ID
* @returns {string} The formatted anchor ID
* @private
*/
_createAnchorId(text) {
return text.toLowerCase().replace(/[^a-z0-9]+/g, "-");
}
/**
* Parses the Wikipedia content into sections with titles and content.
* @param {HTMLElement} mainContent - The main content element to parse
* @returns {Section[]} An array of parsed sections
* @private
*/
parseSections(mainContent) {
const sections = [];
let currentSection = { title: "", content: "" };
let inLeadSection = true;
let currentHtml = "";
mainContent.childNodes.forEach((element) => {
const htmlElement = element;
if (!htmlElement.tagName) return;
if (htmlElement.tagName.match(/^H[1-6]$/i)) {
if (currentHtml) {
if (inLeadSection) {
sections.push({
title: "Introduction",
level: 2,
id: "introduction",
content: this._turndown.convert(currentHtml)
});
inLeadSection = false;
} else {
const content = this._turndown.convert(currentHtml);
sections.push({
title: currentSection.title,
content
});
}
currentHtml = "";
}
const headingText = htmlElement.querySelector(".mw-headline")?.textContent || htmlElement.textContent?.trim();
const level = htmlElement.tagName && htmlElement.tagName[1] ? parseInt(htmlElement.tagName[1]) : 5;
const headingId = this._createAnchorId(headingText);
currentSection = {
title: headingText ?? "",
level,
id: headingId ?? "",
content: ""
};
} else {
currentHtml += htmlElement.outerHTML || "";
}
});
if (currentHtml) {
const content = this._turndown.convert(currentHtml);
sections.push({
...currentSection,
title: currentSection.title,
content
});
}
return sections;
}
/**
* Generates a table of contents from the document headings.
* @param {HTMLElement} root - The root element containing headings
* @returns {string} Markdown formatted table of contents
* @private
*/
generateTableOfContents(root) {
let toc = "## Table of Contents\n\n";
root?.querySelectorAll(".mw-heading").forEach((el) => {
const headingClassname = el.classList.toString();
const headingLevel = headingClassname.endsWith("2") ? 1 : 2;
const headingText = el?.textContent?.trim() ?? "";
const headingId = this._createAnchorId(headingText);
toc += `${" ".repeat(headingLevel)} - [${headingText}](#${headingId.toLowerCase().replace(/\s+/g, "-")})
`;
});
return toc + "\n";
}
/**
* Generates the final Markdown document from the parsed content.
* @param {{title: string, sections: Section[]}} content - The parsed document content
* @param {HTMLElement} root - The root HTML element
* @returns {string} The complete Markdown document
* @private
*/
generateMarkdown(content, root) {
let markdown = "";
markdown += `# ${content.title}
`;
markdown += this.generateTableOfContents(root);
content.sections.forEach((section) => {
if (section.title) {
const headingLevel = "#".repeat(Math.min(section.level ?? 0, 6));
markdown += `${headingLevel} [${section.title}](#${section.id})
`;
}
markdown += section.content + "\n\n";
});
return markdown;
}
};
// src/converters/youtube.ts
var import_fs4 = __toESM(require("fs"), 1);
var import_node_html_parser4 = require("node-html-parser");
var YouTubeConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts a YouTube video page to Markdown format.
* Extracts video title, metadata, description, and transcript (if available).
*
* @param {string} localPath - The local file path to the YouTube page HTML file
* @param {ConversionOptions} options - Conversion options including file extension and URL
* @returns {Promise<DocumentConverterResult>} Object containing the converted markdown content, or null if the file is not a YouTube video page
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (![".html", ".htm"].includes(extension.toLowerCase())) {
return null;
}
const url = options.url || "";
if (!url.startsWith("https://www.youtube.com/watch?") && !url.startsWith("https://www.youtube.com/shorts/")) {
return null;
}
const htmlContent = import_fs4.default.readFileSync(localPath, "utf-8");
const root = (0, import_node_html_parser4.parse)(htmlContent);
const metadata = {
title: root.querySelector("title")?.text || ""
};
root.querySelectorAll("meta").forEach((meta) => {
const attr = meta.getAttribute("property") || meta.getAttribute("name") || meta.getAttribute("itemprop");
if (attr && attr in metadata) {
metadata[attr] = meta.getAttribute("content") || "";
}
});
try {
root.querySelectorAll("script").forEach((script) => {
const content = script.text;
if (content && content.includes("ytInitialData")) {
const match = content.match(/var ytInitialData = ({.+?});/);
if (match && match[1]) {
try {
const data = JSON.parse(match[1]);
const attrdesc = this._findKey(
data,
"attributedDescriptionBodyText"
);
if (attrdesc) {
metadata["description"] = attrdesc.content;
}
} catch (parseError) {
console.debug("Failed to parse script content:", parseError);
}
}
}
});
} catch (err) {
console.error("Error parsing description:", err);
}
let webpageText = "# YouTube\n";
const title = this._get(metadata, ["title", "og:title", "name"]) || "";
if (title) {
webpageText += `
## ${title}
`;
}
let stats = "";
const views = this._get(metadata, ["interactionCount"]);
if (views) stats += `- **Views:** ${views}
`;
const keywords = this._get(metadata, ["keywords"]);
if (keywords) stats += `- **Keywords:** ${keywords}
`;
const runtime = this._get(metadata, ["duration"]);
if (runtime) stats += `- **Runtime:** ${runtime}
`;
if (stats) webpageText += `
### Video Metadata
${stats}
`;
const description = this._get(metadata, ["description", "og:description"]);
if (description) {
webpageText += `
### Description
${description}
`;
}
let transcriptText = "";
const parsedUrl = new URL(url);
const videoId = parsedUrl.searchParams.get("v");
const shortId = parsedUrl.pathname.split("/").pop();
if (videoId || shortId) {
try {
await this.retryOperation(
async () => {
const response = await options.requestsSession?.get(
`https://www.youtubetranscript.com/?server_vid2=${videoId || shortId}`
);
const transcript = (0, import_node_html_parser4.parse)(response?.data);
const transcriptTexts = [];
transcript.querySelectorAll("transcript text").forEach((text) => {
transcriptTexts.push(text.innerText);
});
transcriptText = transcriptTexts.join(" ");
},
3,
2e3
);
} catch (err) {
console.error("Error fetching transcript after retries:", err);
}
}
if (transcriptText) {
webpageText += `
### Transcript
${transcriptText}
`;
}
return {
title,
textContent: webpageText
};
}
/**
* Retries an operation with specified number of attempts and delay between retries
*
* @param {Function} operation - The async function to retry
* @param {number} retries - Number of retry attempts
* @param {number} delay - Delay between retries in milliseconds
* @returns {Promise<any>} Result of the operation if successful
* @throws {Error} If all retry attempts fail
* @private
*/
async retryOperation(operation, retries = 3, delay = 2e3) {
let lastError;
for (let attempt = 0; attempt < retries; attempt++) {
try {
return await operation();
} catch (error) {
const err = error;
console.log(`Attempt ${attempt + 1} failed: ${err.message}`);
lastError = err;
if (attempt < retries - 1) {
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
throw new Error(
`Operation failed after ${retries} attempts: ${lastError?.message}`
);
}
/**
* Retrieves a value from metadata using a list of possible keys.
*
* @param {Record<string, any>} metadata - The metadata object to search in
* @param {string[]} keys - Array of possible keys to look for
* @param {any} defaultValue - Value to return if no key is found
* @returns {any} The first found value or the default value
* @private
*/
_get(metadata, keys, defaultValue = null) {
for (const key of keys) {
if (metadata[key]) {
return metadata[key];
}
}
return defaultValue;
}
/**
* Recursively searches for a specific key in a nested object or array.
*
* @param {any} obj - The object or array to search in
* @param {string} key - The key to search for
* @returns {any} The value associated with the key if found, null otherwise
* @private
*/
_findKey(obj, key) {
if (Array.isArray(obj)) {
for (const item of obj) {
const result = this._findKey(item, key);
if (result) return result;
}
} else if (typeof obj === "object" && obj !== null) {
for (const k in obj) {
if (k === key) return obj[k];
const result = this._findKey(obj[k], key);
if (result) return result;
}
}
return null;
}
};
// src/converters/bingSerp.ts
var import_fs5 = __toESM(require("fs"), 1);
var import_node_html_parser5 = require("node-html-parser");
var BingSerpConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts a Bing search results page to markdown format.
* Only processes HTML files from Bing search URLs.
*
* @param {string} localPath - Path to the local HTML file
* @param {ConversionOptions} options - Conversion options
* @param {string} [options.fileExtension] - File extension (must be .html or .htm)
* @param {string} [options.url] - Original URL (must be a Bing search URL)
* @returns {Promise<DocumentConverterResult>} Conversion result containing search results
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (![".html", ".htm"].includes(extension.toLowerCase())) {
return null;
}
const url = options.url || "";
if (!/^https:\/\/www\.bing\.com\/search\?q=/.test(url)) {
return null;
}
const parsedParams = new URL(url).searchParams;
const query = parsedParams.get("q") || "";
const html = import_fs5.default.readFileSync(localPath, "utf-8");
const root = (0, import_node_html_parser5.parse)(html);
root.querySelectorAll(".tptt").forEach((el) => {
el.textContent = el.textContent + " ";
});
root.querySelectorAll(".algoSlug_icon").forEach((el) => {
el.remove();
});
let results = [];
root.querySelectorAll(".b_algo").forEach((el) => {
let resultText = el.textContent.trim();
results.push(resultText);
});
let webpageText = `## A Bing search for '${query}' found the following results:
`;
webpageText += results.join("\n\n");
return {
title: root.querySelector("title")?.innerText || null,
textContent: webpageText
};
}
};
// src/converters/docx.ts
var import_fs6 = __toESM(require("fs"), 1);
var import_mammoth = __toESM(require("mammoth"), 1);
var DocxConverter = class extends HtmlConverter {
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts a DOCX file to Markdown format.
* Uses Mammoth.js to convert DOCX to HTML, then processes the HTML to Markdown.
*
* @param {string} localPath - Path to the local DOCX file
* @param {ConversionOptions} options - Conversion options
* @param {string} [options.fileExtension] - File extension (must be .docx)
* @param {Array<string>} [options.styleMap] - Custom style mappings for Mammoth.js conversion
* @returns {Promise<DocumentConverterResult>} Conversion result or null if:
* - File is not a DOCX
* - File cannot be read
* - Conversion fails
*
* @throws {Error} If file reading or conversion process fails
* @override
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (extension.toLowerCase() !== ".docx") {
return null;
}
try {
const buffer = await import_fs6.default.promises.readFile(localPath);
const styleMap = options.styleMap;
const result = await import_mammoth.default.convertToHtml({ buffer }, { styleMap });
const htmlContent = result.value;
return this._convert(htmlContent);
} catch (error) {
console.error("Error converting DOCX file:", error);
return null;
}
}
};
// src/converters/xlsx.ts
var import_xlsx = __toESM(require("xlsx"), 1);
var XlsxConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts an Excel file to Markdown format.
* Each sheet is represented as a separate section with a Markdown table.
*
* @override
* @param {string} localPath - The local file path to the Excel file
* @param {ConversionOptions} options - Conversion options including file extension
* @returns {Promise<DocumentConverterResult>} Object containing the converted markdown content, or returns null if the file is not an Excel file
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (![".xlsx", ".xls"].includes(extension.toLowerCase())) {
return null;
}
const workbook = import_xlsx.default.readFile(localPath, { bookVBA: true });
let mdContent = "";
workbook.SheetNames.forEach((sheetName) => {
const worksheet = workbook.Sheets[sheetName];
if (!worksheet) return;
mdContent += `## ${sheetName}
`;
const tableContent = import_xlsx.default.utils.sheet_to_json(worksheet, {});
const firstRow = tableContent[0];
const tableHead = Object.keys(firstRow);
const tableBody = tableContent.map((row) => {
const tableRow = Object.values(row).map((cell) => `${cell}`).join(" | ");
return tableRow;
});
const mdTable = [
// Header row with column names
`|${tableHead.join("|")}`,
// Separator row with dashes
`${tableHead.map((key) => "-".repeat(key.length)).join("|")}`,
// Data rows
`${tableBody.map((row) => `|${row}|`).join("\n")}`
].join("\n");
mdContent += mdTable + "\n\n";
});
return {
title: null,
textContent: mdContent.trim()
};
}
};
// src/converters/pptx.ts
var import_node_pptx_parser = __toESM(require("node-pptx-parser"), 1);
var PptxParser = "default" in import_node_pptx_parser.default ? import_node_pptx_parser.default.default : import_node_pptx_parser.default;
var PptxConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts a PPTX file to markdown format.
* Extracts text from each slide and organizes them in order.
* Slides are sorted by their ID to maintain presentation order.
*
* @param {string} localPath - Path to the PPTX file
* @param {ConversionOptions} options - Conversion options including file extension
* @returns {Promise<DocumentConverterResult>} Object containing formatted markdown as textContent (title is null), or returns null for unsupported file types (.ppt files or non-PowerPoint files)
* @throws {Error} If the file cannot be read or parsed
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (extension.toLowerCase() === ".ppt") {
console.warn("PPT files are not supported. Please use PPTX files.");
return null;
}
if (extension.toLowerCase() !== ".pptx") {
return null;
}
const pptxParser = new PptxParser(localPath);
const pptxSlides = await pptxParser.extractText();
const sortedSlides = pptxSlides.sort((a, b) => {
const aId = a.id.substring(3);
const bId = b.id.substring(3);
return Number(aId) - Number(bId);
});
let mdContent = "";
sortedSlides.forEach((slide, index) => {
const slideName = slide.path?.split("/").pop()?.split(".")[0] || `Slide ${index}`;
mdContent += `
## ${slideName} - ${slide.id}
`;
mdContent += slide.text ? slide.text : "Empty Slide...";
});
return {
title: null,
textContent: mdContent.replace(/\n\n\n/g, "\n").trim()
};
}
};
// src/converters/audio.ts
var import_fs7 = __toESM(require("fs"), 1);
// src/converters/media.ts
var import_exiftool_vendored = require("exiftool-vendored");
var MediaConverter = class extends DocumentConverter {
constructor(priority = DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT) {
super(priority);
}
/**
* Converts a local file to the target format.
* @abstract
* @param {string} localPath - The path to the local file to convert
* @param {ConversionOptions} [options] - Optional conversion configuration
* @returns {Promise<DocumentConverterResult>} A promise that resolves with the conversion result
* @throws {Error} May throw implementation-specific errors during conversion
*/
convert(localPath, options) {
throw new Error("Method not implemented.");
}
/**
* Generates media metadata using exiftool.
* @param {string} localPath - Path to the local audio file
* @returns {Promise<Record<string, any> | null>} The transcription text or null if transcription fails or is unavailable
*/
async _getMetadata(localPath) {
try {
const metadata = await import_exiftool_vendored.exiftool.read(localPath);
return metadata;
} catch (error) {
console.error("Error reading metadata:", error);
return null;
} finally {
import_exiftool_vendored.exiftool.end();
}
}
};
// src/converters/audio.ts
var AudioConverter = class extends MediaConverter {
/**
* Converts an audio file to markdown format, including metadata and transcription.
* @param {string} localPath - Path to the local audio file
* @param {ConversionOptions} options - Conversion options including file extension and LLM callback
* @param {string} options.fileExtension - The file extension of the audio file
* @param {LlmCall} [options.llmCall] - Callback function for audio transcription
* @returns {Promise<DocumentConverterResult>} The conversion result or null if file type not supported
* @override
*/
async convert(localPath, options) {
const supportedAudioExtensions = [".m4a", ".mp3", ".mpga", ".wav"];
if (!supportedAudioExtensions.includes(options.fileExtension.toLowerCase()))
return null;
let mdContent = "";
const metadata = await this._getMetadata(localPath);
if (metadata) {
[
"Title",
"Artist",
"Author",
"Band",
"Album",
"Genre",
"Track",
"DateTimeOriginal",
"CreateDate",
"Duration"
].forEach((field) => {
if (metadata[field]) {
mdContent += `- **${field}**: ${metadata[field]}
`;
}
});
}
try {
const transcript = await this._transcribeAudio(
localPath,
options.llmCall
);
mdContent += `
# Audio Transcript:
${transcript || "[No speech detected]"}`;
} catch (err) {
console.error(err);
mdContent += "\n## Audio Transcript:\nError. Could not transcribe this audio.";
}
return {
title: null,
textContent: mdContent.trim()
};
}
/**
* Transcribes audio content using the provided LLM callback function.
* @param {string} localPath - Path to the local audio file
* @param {LlmCall} llmCall - Callback function for audio transcription
* @returns {Promise<string | null>} The transcription text or null if transcription fails or is unavailable
*/
async _transcribeAudio(localPath, llmCall) {
if (typeof llmCall !== "function") return null;
try {
const file = import_fs7.default.createReadStream(localPath);
const response = await llmCall({ file });
return response;
} catch (error) {
console.error("Error in transcription:", error);
return null;
}
}
};
// src/converters/video.ts
var import_fs8 = __toESM(require("fs"), 1);
var import_tmp = __toESM(require("tmp"), 1);
var import_fluent_ffmpeg = __toESM(require("fluent-ffmpeg"), 1);
var VideoConverter = class extends AudioConverter {
/**
* Converts a video file to markdown format.
* Extracts available metadata and optionally transcribes audio content if FFmpeg is available.
*
* @param {string} localPath - Path to the video file
* @param {ConversionOptions} options - Conversion options including:
* - fileExtension: The file extension (must be .mp4, .mkv, .webm, or .mpeg)
* - llmCall: Optional function for audio transcription
* @returns {Promise<DocumentConverterResult>} Object containing extracted metadata and optional transcript,or returns null for unsupported file types
* @throws {Error} If file processing or transcription fails
*/
async convert(localPath, options) {
const supportedVideoExtensions = [".mp4", ".mkv", ".webm", ".mpeg"];
if (!supportedVideoExtensions.includes(options.fileExtension)) return null;
let mdContent = "";
const metadata = await this._getMetadata(localPath);
if (metadata) {
[
"Title",
"Artist",
"Author",
"Band",
"Album",
"Genre",
"Track",
"DateTimeOriginal",
"CreateDate",
"Duration"
].forEach((field) => {
if (metadata[field]) {
mdContent += `- **${field}**: ${metadata[field]}
`;
}
});
}
if (typeof options.llmCall === "function" && global.IS_FFMPEG_CAPABLE) {
const tempPath = import_tmp.default.tmpNameSync({
prefix: "markitdownjs-",
postfix: ".wav"
});
try {
await new Promise((resolve, reject) => {
(0, import_fluent_ffmpeg.default)(localPath).toFormat("wav").save(tempPath).on("end", resolve).on("error", reject);
}).catch((err) => {
console.error("Error in video conversion:", err);
});
const transcript = await super._transcribeAudio(
localPath,
options.llmCall
);
mdContent += `
### Audio Transcript:
${transcript || "[No speech detected]"}`;
} catch (err) {
mdContent += "\n\n### Audio Transcript:\nError. Could not transcribe this audio.";
} finally {
import_fs8.default.unlinkSync(tempPath);
}
}
return { title: null, textContent: mdContent.trim() };
}
};
// src/converters/image.ts
var import_fs9 = __toESM(require("fs"), 1);
var import_mime_types2 = __toESM(require("mime-types"), 1);
var import_node_tesseract_ocr = __toESM(require("node-tesseract-ocr"), 1);
var ImageConverter = class extends MediaConverter {
/**
* Converts an image file to markdown format with metadata, OCR text, and AI description.
*
* @param {string} localPath - Path to the local image file
* @param {ConversionOptions} options - Conversion options
* @param {string} [options.fileExtension] - File extension (must be .jpg, .jpeg, or .png)
* @param {LlmCall} [options.llmCall] - Callback function for LLM image description
* @returns {Promise<DocumentConverterResult>} Conversion result or null if:
* - File is not a supported image type
* - File cannot be read
*
* @remarks
* The converter attempts to extract three types of information:
* 1. Metadata fields: ImageSize, Title, Caption, Description, Keywords, Artist,
* Author, DateTimeOriginal, CreateDate, GPSPosition
* 2. OCR text (requires Tesseract installation)
* 3. AI-generated description (requires configured llmCall)
* @override
*/
async convert(localPath, options) {
const extension = options.fileExtension || "";
if (![".jpg", ".jpeg", ".png"].includes(extension.toLowerCase())) {
return null;
}
let mdContent = "";
const metadata = await this._getMetadata(localPath);
if (metadata) {
const fields = [
"ImageSize",
"Title",
"Caption",
"Description",
"Keywords",
"Artist",
"Author",
"DateTimeOriginal",
"CreateDate",
"GPSPosition"
];
fields.forEach((field) => {
if (metadata[field]) {
mdContent += `- **${field}**: ${metadata[field]}
`;
}
});
}
try {
const text = await import_node_tesseract_ocr.default.recognize(localPath);
if (text) mdContent += `
# Text:
${text.trim()}`;
} catch (error) {
}
const description = awai