@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
443 lines (442 loc) • 15 kB
JavaScript
/**
* HTML to Markdown conversion for document processors.
*
* Scoped deliberately to the small, well-formed HTML subset that document
* converters emit (mammoth for DOCX): headings, paragraphs, emphasis, links,
* images, lists, block quotes, preformatted text and tables. It is not a
* general-purpose web-page converter, which is why the repository gains no
* dependency for it.
*
* @module utils/htmlToMarkdown
*/
/** Tags that never have a closing tag and so never open a scope. */
const VOID_TAGS = new Set([
"area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"link",
"meta",
"source",
"track",
"wbr",
]);
const NAMED_ENTITIES = {
amp: "&",
lt: "<",
gt: ">",
quot: '"',
apos: "'",
nbsp: " ",
};
/**
* Decode the numeric and named character references a document converter can
* emit. Unknown references are left verbatim rather than dropped, so no text
* is silently lost.
*/
function decodeEntities(value) {
return value.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, body) => {
if (body.startsWith("#")) {
const codePoint = body.startsWith("#x") || body.startsWith("#X")
? Number.parseInt(body.slice(2), 16)
: Number.parseInt(body.slice(1), 10);
if (Number.isNaN(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {
return match;
}
try {
return String.fromCodePoint(codePoint);
}
catch {
return match;
}
}
const named = NAMED_ENTITIES[body.toLowerCase()];
return named ?? match;
});
}
/**
* Parse the HTML subset into a node tree.
*
* Unbalanced closing tags are ignored and unclosed elements are closed at the
* end of input, so malformed markup degrades to partial structure instead of
* throwing.
*/
function parseHtml(html) {
const root = {
kind: "element",
tag: "#root",
attrs: {},
children: [],
};
const stack = [root];
const tagPattern = /<\/?([a-zA-Z][a-zA-Z0-9-]*)((?:[^>"']|"[^"]*"|'[^']*')*)>/g;
let cursor = 0;
let match;
const pushText = (raw) => {
if (!raw) {
return;
}
const current = stack[stack.length - 1];
current.children.push({ kind: "text", value: decodeEntities(raw) });
};
while ((match = tagPattern.exec(html)) !== null) {
pushText(html.slice(cursor, match.index));
cursor = match.index + match[0].length;
const tag = match[1].toLowerCase();
const isClosing = match[0].startsWith("</");
const isSelfClosing = match[2].trimEnd().endsWith("/");
if (isClosing) {
// Only unwind if the tag is actually open, so a stray `</div>` is
// inert. Search from the top of the stack down so a closing tag
// matches the innermost (most recently opened) element with that
// name — `findIndex` would match the OUTERMOST one instead, and for
// nested same-tag elements (`<div><div>A</div>B</div>`) that pops both
// levels at once, reattaching everything after the real inner close to
// the wrong ancestor.
let openIndex = -1;
for (let i = stack.length - 1; i >= 0; i--) {
if (stack[i].tag === tag) {
openIndex = i;
break;
}
}
if (openIndex > 0) {
stack.length = openIndex;
}
continue;
}
const element = {
kind: "element",
tag,
attrs: parseAttributes(match[2]),
children: [],
};
stack[stack.length - 1].children.push(element);
if (!isSelfClosing && !VOID_TAGS.has(tag)) {
stack.push(element);
}
}
pushText(html.slice(cursor));
return root.children;
}
function parseAttributes(raw) {
const attrs = {};
const pattern = /([a-zA-Z_:][a-zA-Z0-9_.:-]*)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>]+))/g;
let match;
while ((match = pattern.exec(raw)) !== null) {
const value = match[3] ?? match[4] ?? match[5] ?? "";
attrs[match[1].toLowerCase()] = decodeEntities(value);
}
return attrs;
}
function isElement(node) {
return node.kind === "element";
}
/** Collapse runs of whitespace the way an HTML renderer would. */
function collapseWhitespace(value) {
return value
.replace(/\s+/g, " ")
.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">");
}
/**
* Format a link/image destination per CommonMark's angle-bracket form.
*
* A bare `(dest)` destination ends at the first whitespace or unbalanced
* `)`, so a value containing either — an `href`/`src` with a space, or one
* with unbalanced parentheses — would silently truncate or corrupt the
* surrounding `[text](dest)` / `` markup. Wrapping in `<...>`
* makes the destination unambiguous; CommonMark requires any literal `<`
* or `>` inside that form to be escaped. Ordinary destinations (no
* whitespace or parens) are returned byte-identical to before.
*/
function formatLinkDestination(dest) {
if (!/[\s()]/.test(dest)) {
return dest;
}
// Backslash first: escaping only `<`/`>` turns `\>` into `\\>`, which
// CommonMark reads as a literal backslash followed by a bare `>`.
const escaped = dest
.replace(/\\/g, "\\\\")
.replace(/</g, "\\<")
.replace(/>/g, "\\>");
return `<${escaped}>`;
}
/** Render inline content (emphasis, links, images, line breaks) to markdown. */
function renderInline(nodes) {
let out = "";
for (const node of nodes) {
if (node.kind === "text") {
out += collapseWhitespace(node.value);
continue;
}
switch (node.tag) {
case "script":
case "style":
break;
case "strong":
case "b": {
const inner = renderInline(node.children).trim();
out += inner ? `**${inner}**` : "";
break;
}
case "em":
case "i": {
const inner = renderInline(node.children).trim();
out += inner ? `*${inner}*` : "";
break;
}
case "code": {
const inner = renderInline(node.children).trim();
out += inner ? `\`${inner}\`` : "";
break;
}
case "a": {
const inner = renderInline(node.children).trim();
const href = node.attrs.href;
out += href ? `[${inner}](${formatLinkDestination(href)})` : inner;
break;
}
case "img": {
const alt = node.attrs.alt ?? "";
const src = node.attrs.src ?? "";
// A DOCX image is inlined as a multi-megabyte data URI; keep the
// alt text and drop the payload rather than flooding the prompt.
out += src.startsWith("data:")
? ``
: `})`;
break;
}
case "br":
out += "\n";
break;
default:
out += renderInline(node.children);
break;
}
}
return out;
}
/**
* Extract a `<pre>` block's text content verbatim.
*
* `renderInline()` funnels every text node through `collapseWhitespace()`,
* which does `.replace(/\s+/g, " ")` — right for prose, destructive for
* code: it collapses every newline, tab and indentation run onto one line.
* This walks the same tree but keeps each text node's raw value, so a
* multi-line, indented code sample survives into the fenced block intact.
* A nested element (e.g. `<pre><code>…</code></pre>`) still contributes its
* text, but ignores inline markdown semantics like `**`/`*`/`` ` `` —
* `<pre>` content is literal code, not prose, and injecting markdown markers
* into it would corrupt the code rather than format it.
*/
function renderPreformattedText(nodes) {
let out = "";
for (const node of nodes) {
if (node.kind === "text") {
out += node.value;
continue;
}
if (node.tag === "script" || node.tag === "style") {
continue;
}
if (node.tag === "br") {
out += "\n";
continue;
}
out += renderPreformattedText(node.children);
}
return out;
}
/** Collect the `<tr>` rows under a table, seeing through thead/tbody/tfoot. */
function collectRows(node) {
const rows = [];
for (const child of node.children.filter(isElement)) {
if (child.tag === "tr") {
rows.push(child);
}
else if (["thead", "tbody", "tfoot"].includes(child.tag)) {
rows.push(...collectRows(child));
}
}
return rows;
}
/** A pipe inside a cell would otherwise split it into two columns. */
function escapeCell(value) {
return value
.replace(/\\/g, "\\\\")
.replace(/\|/g, "\\|")
.replace(/\n/g, " ")
.trim();
}
function renderTable(node) {
const rows = collectRows(node);
if (rows.length === 0) {
return "";
}
const cellsOf = (row) => row.children
.filter(isElement)
.filter((cell) => cell.tag === "td" || cell.tag === "th")
.map((cell) => escapeCell(renderBlocks(cell.children).replace(/\n+/g, " ")));
// Markdown has no headerless table, so when the source marked no header row
// the first row is promoted — that is what a reader sees anyway.
const header = cellsOf(rows[0]);
const bodyRows = rows.slice(1).map(cellsOf);
const columnCount = Math.max(header.length, ...bodyRows.map((r) => r.length), 1);
const pad = (cells) => Array.from({ length: columnCount }, (_, i) => cells[i] ?? "");
const lines = [
`| ${pad(header).join(" | ")} |`,
`| ${Array.from({ length: columnCount }, () => "---").join(" | ")} |`,
...bodyRows.map((cells) => `| ${pad(cells).join(" | ")} |`),
];
return lines.join("\n");
}
function renderList(node, depth) {
const ordered = node.tag === "ol";
const items = node.children
.filter(isElement)
.filter((child) => child.tag === "li");
const indent = " ".repeat(depth);
return items
.map((item, index) => {
const marker = ordered ? `${index + 1}.` : "-";
// Nested lists render themselves at the next depth; everything else on
// the item is inline content belonging to this bullet.
const nested = item.children
.filter(isElement)
.filter((child) => child.tag === "ul" || child.tag === "ol");
const own = item.children.filter((child) => !(isElement(child) && (child.tag === "ul" || child.tag === "ol")));
const label = renderBlocks(own).replace(/\n+/g, " ").trim();
const nestedText = nested
.map((child) => renderList(child, depth + 1))
.filter(Boolean)
.join("\n");
return [`${indent}${marker} ${label}`.trimEnd(), nestedText]
.filter(Boolean)
.join("\n");
})
.join("\n");
}
/** Render block-level content, separating blocks with a blank line. */
function renderBlocks(nodes) {
const blocks = [];
let inlineBuffer = "";
const flushInline = () => {
const text = inlineBuffer.trim();
if (text) {
blocks.push(text);
}
inlineBuffer = "";
};
for (const node of nodes) {
if (node.kind === "text") {
inlineBuffer += collapseWhitespace(node.value);
continue;
}
const headingMatch = /^h([1-6])$/.exec(node.tag);
if (headingMatch) {
flushInline();
const level = Number.parseInt(headingMatch[1], 10);
const text = renderInline(node.children).replace(/\n+/g, " ").trim();
if (text) {
blocks.push(`${"#".repeat(level)} ${text}`);
}
continue;
}
switch (node.tag) {
case "script":
case "style":
flushInline();
break;
case "p": {
flushInline();
const text = renderBlocks(node.children).trim();
if (text) {
blocks.push(text);
}
break;
}
case "ul":
case "ol": {
flushInline();
const text = renderList(node, 0);
if (text) {
blocks.push(text);
}
break;
}
case "table": {
flushInline();
const text = renderTable(node);
if (text) {
blocks.push(text);
}
break;
}
case "blockquote": {
flushInline();
const inner = renderBlocks(node.children).trim();
if (inner) {
blocks.push(inner
.split("\n")
.map((line) => (line ? `> ${line}` : ">"))
.join("\n"));
}
break;
}
case "pre": {
flushInline();
const inner = renderPreformattedText(node.children).replace(/^\n+|\n+$/g, "");
if (inner) {
blocks.push(`\`\`\`\n${inner}\n\`\`\``);
}
break;
}
case "hr": {
flushInline();
blocks.push("---");
break;
}
case "div":
case "section":
case "article": {
flushInline();
const inner = renderBlocks(node.children).trim();
if (inner) {
blocks.push(inner);
}
break;
}
default:
inlineBuffer += renderInline([node]);
break;
}
}
flushInline();
return blocks.join("\n\n");
}
/**
* Convert an HTML fragment to Markdown.
*
* @param html - HTML produced by a document converter
* @returns Markdown text, or an empty string when the fragment has no content
*
* @example
* ```typescript
* htmlToMarkdown("<h1>Title</h1><p>Body</p>"); // "# Title\n\nBody"
* ```
*/
export function htmlToMarkdown(html) {
if (!html || !html.trim()) {
return "";
}
return renderBlocks(parseHtml(html))
.replace(/[ \t]+$/gm, "")
.replace(/\n{3,}/g, "\n\n")
.trim();
}