UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

912 lines (911 loc) 25.9 kB
import { t as MarkdownIt } from "./markdown-it-BzqOxpTv.js"; //#region packages/markdown-core/src/chunk-text.ts function resolveChunkEarlyReturn(text, limit) { if (!text) return []; if (limit <= 0) return [text]; if (text.length <= limit) return [text]; } function scanParenAwareBreakpoints(text) { let lastNewline = -1; let lastWhitespace = -1; let depth = 0; for (let i = 0; i < text.length; i++) { const char = text[i]; if (char === "(") { depth += 1; continue; } if (char === ")" && depth > 0) { depth -= 1; continue; } if (depth !== 0) continue; if (char === "\n") lastNewline = i; else if (/\s/.test(char)) lastWhitespace = i; } return { lastNewline, lastWhitespace }; } /** * Splits plain text into size-bounded chunks at readable boundaries. * * Returns the original text as one chunk when the limit is non-positive. */ function chunkText(text, limit) { const early = resolveChunkEarlyReturn(text, limit); if (early) return early; const chunks = []; let cursor = 0; while (cursor < text.length) { if (text.length - cursor <= limit) { chunks.push(text.slice(cursor)); break; } const windowEnd = Math.min(text.length, cursor + limit); const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(text.slice(cursor, windowEnd)); const breakOffset = lastNewline > 0 ? lastNewline : lastWhitespace; const end = breakOffset > 0 ? cursor + breakOffset : windowEnd; chunks.push(text.slice(cursor, end)); cursor = end; while (cursor < text.length && /\s/.test(text[cursor] ?? "")) cursor += 1; } return chunks; } //#endregion //#region packages/markdown-core/src/ir.ts function createStyleSpan(params) { const span = { start: params.start, end: params.end, style: params.style }; if (params.language) span.language = params.language; return span; } function createMarkdownIt(options) { const md = new MarkdownIt({ html: false, linkify: options.linkify ?? true, breaks: false, typographer: false }); md.enable("strikethrough"); if (options.tableMode && options.tableMode !== "off") md.enable("table"); else md.disable("table"); if (options.autolink === false) md.disable("autolink"); return md; } function getAttr(token, name) { if (token.attrGet) return token.attrGet(name); if (token.attrs) { for (const [key, value] of token.attrs) if (key === name) return value; } return null; } function createTextToken(base, content) { return { ...base, type: "text", content, children: void 0 }; } function applySpoilerTokens(tokens) { for (const token of tokens) if (token.children && token.children.length > 0) token.children = injectSpoilersIntoInline(token.children); } function injectSpoilersIntoInline(tokens) { let totalDelims = 0; for (const token of tokens) { if (token.type !== "text") continue; const content = token.content ?? ""; let i = 0; while (i < content.length) { const next = content.indexOf("||", i); if (next === -1) break; totalDelims += 1; i = next + 2; } } if (totalDelims < 2) return tokens; const usableDelims = totalDelims - totalDelims % 2; const result = []; const state = { spoilerOpen: false }; let consumedDelims = 0; for (const token of tokens) { if (token.type !== "text") { result.push(token); continue; } const content = token.content ?? ""; if (!content.includes("||")) { result.push(token); continue; } let index = 0; while (index < content.length) { const next = content.indexOf("||", index); if (next === -1) { if (index < content.length) result.push(createTextToken(token, content.slice(index))); break; } if (consumedDelims >= usableDelims) { result.push(createTextToken(token, content.slice(index))); break; } if (next > index) result.push(createTextToken(token, content.slice(index, next))); consumedDelims += 1; state.spoilerOpen = !state.spoilerOpen; result.push({ type: state.spoilerOpen ? "spoiler_open" : "spoiler_close" }); index = next + 2; } } return result; } function initRenderTarget() { return { text: "", styles: [], openStyles: [], links: [], linkStack: [] }; } function resolveRenderTarget(state) { return state.table?.currentCell ?? state; } function appendText(state, value) { if (!value) return; const target = resolveRenderTarget(state); target.text += value; } function openStyle(state, style) { const target = resolveRenderTarget(state); target.openStyles.push({ style, start: target.text.length }); } function closeStyle(state, style, options) { const target = resolveRenderTarget(state); for (let i = target.openStyles.length - 1; i >= 0; i -= 1) if (target.openStyles[i]?.style === style) { const start = target.openStyles[i].start; target.openStyles.splice(i, 1); const end = options?.trimTrailingParagraphSeparator && target.text.endsWith("\n\n") ? target.text.length - 2 : target.text.length; if (end > start) target.styles.push({ start, end, style }); return; } } function appendParagraphSeparator(state, token) { if (state.table) return; if (state.env.listStack.length > 0) { const directListParagraphLevel = (state.env.listStack[state.env.listStack.length - 1]?.openLevel ?? 0) + 2; if (token?.type !== "paragraph_close" || token.hidden || token.level !== directListParagraphLevel) return; } state.text += "\n\n"; } function appendTopLevelListSeparator(state) { if ((state.text.match(/\n*$/)?.[0].length ?? 0) < 2) state.text += "\n"; } function appendNestedListSeparator(state) { if (!state.text.endsWith("\n")) state.text += "\n"; } function appendListPrefix(state) { const stack = state.env.listStack; const top = stack[stack.length - 1]; if (!top) return; top.index += 1; const indent = " ".repeat(Math.max(0, stack.length - 1)); const prefix = top.type === "ordered" ? `${top.index}. ` : "• "; state.text += `${indent}${prefix}`; } function renderInlineCode(state, content) { if (!content) return; const target = resolveRenderTarget(state); const start = target.text.length; target.text += content; target.styles.push({ start, end: start + content.length, style: "code" }); } function resolveFenceLanguage(info) { return info?.trim().split(/\s+/, 1)[0]?.trim() || void 0; } function renderCodeBlock(state, content, info) { let code = content ?? ""; if (!code.endsWith("\n")) code = `${code}\n`; const target = resolveRenderTarget(state); const start = target.text.length; target.text += code; target.styles.push(createStyleSpan({ start, end: start + code.length, style: "code_block", language: resolveFenceLanguage(info) })); if (state.env.listStack.length === 0) target.text += "\n"; } function handleLinkClose(state) { const target = resolveRenderTarget(state); const link = target.linkStack.pop(); if (!link?.href) return; const href = link.href.trim(); if (!href) return; const start = link.labelStart; const end = target.text.length; if (end <= start) { target.links.push({ start, end, href }); return; } target.links.push({ start, end, href }); } function initTableState() { return { headers: [], rows: [], currentRow: [], currentCell: null, inHeader: false }; } function finishTableCell(cell) { closeRemainingStyles(cell); return { text: cell.text, styles: cell.styles, links: cell.links }; } function trimCell(cell) { const text = cell.text; let start = 0; let end = text.length; while (start < end && /\s/.test(text[start] ?? "")) start += 1; while (end > start && /\s/.test(text[end - 1] ?? "")) end -= 1; if (start === 0 && end === text.length) return cell; const trimmedText = text.slice(start, end); const trimmedLength = trimmedText.length; const trimmedStyles = []; for (const span of cell.styles) { const sliceStart = Math.max(0, span.start - start); const sliceEnd = Math.min(trimmedLength, span.end - start); if (sliceEnd > sliceStart) trimmedStyles.push({ start: sliceStart, end: sliceEnd, style: span.style }); } const trimmedLinks = []; for (const span of cell.links) { const sliceStart = Math.max(0, span.start - start); const sliceEnd = Math.min(trimmedLength, span.end - start); if (sliceEnd > sliceStart) trimmedLinks.push({ start: sliceStart, end: sliceEnd, href: span.href }); } return { text: trimmedText, styles: trimmedStyles, links: trimmedLinks }; } function appendCell(state, cell) { if (!cell.text) return; const start = state.text.length; state.text += cell.text; for (const span of cell.styles) state.styles.push({ start: start + span.start, end: start + span.end, style: span.style }); for (const link of cell.links) state.links.push({ start: start + link.start, end: start + link.end, href: link.href }); } function appendCellTextOnly(state, cell) { if (!cell.text) return; state.text += cell.text; } function collectTableBlock(state) { if (!state.table) return; state.collectedTables.push({ headers: state.table.headers.map((cell) => trimCell(cell).text), rows: state.table.rows.map((row) => row.map((cell) => trimCell(cell).text)), placeholderOffset: state.text.length }); } function appendTableBulletValue(state, params) { const { header, value, columnIndex, includeColumnFallback } = params; if (!value?.text) return; state.text += "• "; if (header?.text) { appendCell(state, header); state.text += ": "; } else if (includeColumnFallback) state.text += `Column ${columnIndex}: `; appendCell(state, value); state.text += "\n"; } function renderTableAsBullets(state) { if (!state.table) return; const headers = state.table.headers.map(trimCell); const rows = state.table.rows.map((row) => row.map(trimCell)); if (headers.length === 0 && rows.length === 0) return; if (headers.length > 1 && rows.length > 0) for (const row of rows) { if (row.length === 0) continue; const rowLabel = row[0]; if (rowLabel?.text) { const labelStart = state.text.length; appendCell(state, rowLabel); const labelEnd = state.text.length; if (labelEnd > labelStart) state.styles.push({ start: labelStart, end: labelEnd, style: "bold" }); state.text += "\n"; } for (let i = 1; i < row.length; i++) appendTableBulletValue(state, { header: headers[i], value: row[i], columnIndex: i, includeColumnFallback: true }); state.text += "\n"; } else for (const row of rows) { for (let i = 0; i < row.length; i++) appendTableBulletValue(state, { header: headers[i], value: row[i], columnIndex: i, includeColumnFallback: false }); state.text += "\n"; } } function renderTableAsCode(state) { if (!state.table) return; const headers = state.table.headers.map(trimCell); const rows = state.table.rows.map((row) => row.map(trimCell)); const columnCount = Math.max(headers.length, ...rows.map((row) => row.length)); if (columnCount === 0) return; const widths = Array.from({ length: columnCount }, () => 0); const updateWidths = (cells) => { for (let i = 0; i < columnCount; i += 1) { const width = cells[i]?.text.length ?? 0; if (widths[i] < width) widths[i] = width; } }; updateWidths(headers); for (const row of rows) updateWidths(row); const codeStart = state.text.length; const appendRow = (cells) => { state.text += "|"; for (let i = 0; i < columnCount; i += 1) { state.text += " "; const cell = cells[i]; if (cell) appendCellTextOnly(state, cell); const pad = widths[i] - (cell?.text.length ?? 0); if (pad > 0) state.text += " ".repeat(pad); state.text += " |"; } state.text += "\n"; }; const appendDivider = () => { state.text += "|"; for (let i = 0; i < columnCount; i += 1) { const dashCount = Math.max(3, widths[i]); state.text += ` ${"-".repeat(dashCount)} |`; } state.text += "\n"; }; appendRow(headers); appendDivider(); for (const row of rows) appendRow(row); const codeEnd = state.text.length; if (codeEnd > codeStart) state.styles.push({ start: codeStart, end: codeEnd, style: "code_block" }); if (state.env.listStack.length === 0) state.text += "\n"; } function renderTokens(tokens, state) { for (const token of tokens) switch (token.type) { case "inline": if (token.children) renderTokens(token.children, state); break; case "text": appendText(state, token.content ?? ""); break; case "em_open": openStyle(state, "italic"); break; case "em_close": closeStyle(state, "italic"); break; case "strong_open": openStyle(state, "bold"); break; case "strong_close": closeStyle(state, "bold"); break; case "s_open": openStyle(state, "strikethrough"); break; case "s_close": closeStyle(state, "strikethrough"); break; case "code_inline": renderInlineCode(state, token.content ?? ""); break; case "spoiler_open": if (state.enableSpoilers) openStyle(state, "spoiler"); break; case "spoiler_close": if (state.enableSpoilers) closeStyle(state, "spoiler"); break; case "link_open": { const href = getAttr(token, "href") ?? ""; const target = resolveRenderTarget(state); target.linkStack.push({ href, labelStart: target.text.length }); break; } case "link_close": handleLinkClose(state); break; case "image": appendText(state, token.content ?? ""); break; case "softbreak": case "hardbreak": appendText(state, "\n"); break; case "paragraph_close": appendParagraphSeparator(state, token); break; case "heading_open": if (state.headingStyle === "bold") openStyle(state, "bold"); break; case "heading_close": if (state.headingStyle === "bold") closeStyle(state, "bold"); appendParagraphSeparator(state); break; case "blockquote_open": if (state.blockquotePrefix) state.text += state.blockquotePrefix; openStyle(state, "blockquote"); break; case "blockquote_close": closeStyle(state, "blockquote", { trimTrailingParagraphSeparator: true }); break; case "bullet_list_open": if (state.env.listStack.length > 0) appendNestedListSeparator(state); state.env.listStack.push({ type: "bullet", index: 0, openLevel: token.level ?? 0 }); break; case "bullet_list_close": state.env.listStack.pop(); if (state.env.listStack.length === 0) appendTopLevelListSeparator(state); break; case "ordered_list_open": { if (state.env.listStack.length > 0) appendNestedListSeparator(state); const start = Number(getAttr(token, "start") ?? "1"); state.env.listStack.push({ type: "ordered", index: start - 1, openLevel: token.level ?? 0 }); break; } case "ordered_list_close": state.env.listStack.pop(); if (state.env.listStack.length === 0) appendTopLevelListSeparator(state); break; case "list_item_open": appendListPrefix(state); break; case "list_item_close": if (!state.text.endsWith("\n")) state.text += "\n"; break; case "code_block": case "fence": renderCodeBlock(state, token.content ?? "", token.info); break; case "html_block": case "html_inline": appendText(state, token.content ?? ""); break; case "table_open": if (state.tableMode !== "off") { state.table = initTableState(); state.hasTables = true; } break; case "table_close": if (state.table) { if (state.tableMode === "bullets") renderTableAsBullets(state); else if (state.tableMode === "code") renderTableAsCode(state); else if (state.tableMode === "block") collectTableBlock(state); } state.table = null; break; case "thead_open": if (state.table) state.table.inHeader = true; break; case "thead_close": if (state.table) state.table.inHeader = false; break; case "tbody_open": case "tbody_close": break; case "tr_open": if (state.table) state.table.currentRow = []; break; case "tr_close": if (state.table) { if (state.table.inHeader) state.table.headers = state.table.currentRow; else state.table.rows.push(state.table.currentRow); state.table.currentRow = []; } break; case "th_open": case "td_open": if (state.table) state.table.currentCell = initRenderTarget(); break; case "th_close": case "td_close": if (state.table?.currentCell) { state.table.currentRow.push(finishTableCell(state.table.currentCell)); state.table.currentCell = null; } break; case "hr": state.text += "───\n\n"; break; default: if (token.children) renderTokens(token.children, state); break; } } function closeRemainingStyles(target) { for (let i = target.openStyles.length - 1; i >= 0; i -= 1) { const open = target.openStyles[i]; const end = target.text.length; if (end > open.start) target.styles.push({ start: open.start, end, style: open.style }); } target.openStyles = []; } function clampStyleSpans(spans, maxLength) { const clamped = []; for (const span of spans) { const start = Math.max(0, Math.min(span.start, maxLength)); const end = Math.max(start, Math.min(span.end, maxLength)); if (end > start) clamped.push(createStyleSpan({ start, end, style: span.style, language: span.language })); } return clamped; } function clampLinkSpans(spans, maxLength) { const clamped = []; for (const span of spans) { const start = Math.max(0, Math.min(span.start, maxLength)); const end = Math.max(start, Math.min(span.end, maxLength)); if (end > start) clamped.push({ start, end, href: span.href }); } return clamped; } function mergeStyleSpans(spans) { const sorted = [...spans].toSorted((a, b) => { if (a.start !== b.start) return a.start - b.start; if (a.end !== b.end) return a.end - b.end; return a.style.localeCompare(b.style); }); const merged = []; for (const span of sorted) { const prev = merged[merged.length - 1]; if (prev && prev.style === span.style && prev.language === span.language && (span.start < prev.end || span.start === prev.end && span.style !== "blockquote")) { prev.end = Math.max(prev.end, span.end); continue; } merged.push({ ...span }); } return merged; } function resolveSliceBounds(span, start, end) { const sliceStart = Math.max(span.start, start); const sliceEnd = Math.min(span.end, end); if (sliceEnd <= sliceStart) return null; return { start: sliceStart, end: sliceEnd }; } function sliceStyleSpans(spans, start, end) { if (spans.length === 0) return []; const sliced = []; for (const span of spans) { const bounds = resolveSliceBounds(span, start, end); if (!bounds) continue; sliced.push(createStyleSpan({ start: bounds.start - start, end: bounds.end - start, style: span.style, language: span.language })); } return mergeStyleSpans(sliced); } function sliceLinkSpans(spans, start, end) { if (spans.length === 0) return []; const sliced = []; for (const span of spans) { const bounds = resolveSliceBounds(span, start, end); if (!bounds) continue; sliced.push({ start: bounds.start - start, end: bounds.end - start, href: span.href }); } return sliced; } function sliceMarkdownIR(ir, start, end) { return { text: ir.text.slice(start, end), styles: sliceStyleSpans(ir.styles, start, end), links: sliceLinkSpans(ir.links, start, end) }; } function markdownToIR(markdown, options = {}) { return markdownToIRWithMeta(markdown, options).ir; } function markdownToIRWithMeta(markdown, options = {}) { const env = { listStack: [] }; const tokens = createMarkdownIt(options).parse(markdown ?? "", env); if (options.enableSpoilers) applySpoilerTokens(tokens); const tableMode = options.tableMode ?? "off"; const state = { text: "", styles: [], openStyles: [], links: [], linkStack: [], env, headingStyle: options.headingStyle ?? "none", blockquotePrefix: options.blockquotePrefix ?? "", enableSpoilers: options.enableSpoilers ?? false, tableMode, table: null, hasTables: false, collectedTables: [] }; renderTokens(tokens, state); closeRemainingStyles(state); const trimmedLength = state.text.trimEnd().length; let codeBlockEnd = 0; for (const span of state.styles) { if (span.style !== "code_block") continue; if (span.end > codeBlockEnd) codeBlockEnd = span.end; } const finalLength = Math.max(trimmedLength, codeBlockEnd); return { ir: { text: finalLength === state.text.length ? state.text : state.text.slice(0, finalLength), styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)), links: clampLinkSpans(state.links, finalLength) }, hasTables: state.hasTables, tables: state.collectedTables.map((table) => Object.assign({}, table, { placeholderOffset: Math.min(table.placeholderOffset, finalLength) })) }; } function chunkMarkdownIR(ir, limit) { if (!ir.text) return []; if (limit <= 0 || ir.text.length <= limit) return [ir]; const chunks = chunkText(ir.text, limit); const results = []; let cursor = 0; chunks.forEach((chunk, index) => { if (!chunk) return; if (index > 0) while (cursor < ir.text.length && /\s/.test(ir.text[cursor] ?? "")) cursor += 1; const start = cursor; const end = Math.min(ir.text.length, start + chunk.length); results.push({ text: chunk, styles: sliceStyleSpans(ir.styles, start, end), links: sliceLinkSpans(ir.links, start, end) }); cursor = end; }); return results; } //#endregion //#region packages/markdown-core/src/render.ts const STYLE_RANK = new Map([ "blockquote", "code_block", "code", "bold", "italic", "strikethrough", "spoiler" ].map((style, index) => [style, index])); function sortStyleSpans(spans) { return [...spans].toSorted((a, b) => { if (a.start !== b.start) return a.start - b.start; if (a.end !== b.end) return b.end - a.end; return (STYLE_RANK.get(a.style) ?? 0) - (STYLE_RANK.get(b.style) ?? 0); }); } /** Renders Markdown IR by nesting configured style markers and optional link markers. */ function renderMarkdownWithMarkers(ir, options) { const text = ir.text ?? ""; if (!text) return ""; const styleMarkers = options.styleMarkers; const styled = sortStyleSpans(ir.styles.filter((span) => Boolean(styleMarkers[span.style]))); const boundaries = /* @__PURE__ */ new Set(); boundaries.add(0); boundaries.add(text.length); const startsAt = /* @__PURE__ */ new Map(); for (const span of styled) { if (span.start === span.end) continue; boundaries.add(span.start); boundaries.add(span.end); const bucket = startsAt.get(span.start); if (bucket) bucket.push(span); else startsAt.set(span.start, [span]); } for (const spans of startsAt.values()) spans.sort((a, b) => { if (a.end !== b.end) return b.end - a.end; return (STYLE_RANK.get(a.style) ?? 0) - (STYLE_RANK.get(b.style) ?? 0); }); const linkStarts = /* @__PURE__ */ new Map(); if (options.buildLink) for (const link of ir.links) { if (link.start === link.end) continue; const rendered = options.buildLink(link, text); if (!rendered) continue; boundaries.add(rendered.start); boundaries.add(rendered.end); const openBucket = linkStarts.get(rendered.start); if (openBucket) openBucket.push(rendered); else linkStarts.set(rendered.start, [rendered]); } const points = [...boundaries].toSorted((a, b) => a - b); const stack = []; let out = ""; for (let i = 0; i < points.length; i += 1) { const pos = points[i]; while (stack.length && stack[stack.length - 1]?.end === pos) { const item = stack.pop(); if (item) out += item.close; } const openingItems = []; const openingLinks = linkStarts.get(pos); if (openingLinks && openingLinks.length > 0) for (const [index, link] of openingLinks.entries()) openingItems.push({ end: link.end, open: link.open, close: link.close, kind: "link", index }); const openingStyles = startsAt.get(pos); if (openingStyles) for (const [index, span] of openingStyles.entries()) { const marker = styleMarkers[span.style]; if (!marker) continue; openingItems.push({ end: span.end, open: typeof marker.open === "function" ? marker.open(span) : marker.open, close: marker.close, kind: "style", style: span.style, index }); } if (openingItems.length > 0) { openingItems.sort((a, b) => { if (a.end !== b.end) return b.end - a.end; if (a.kind !== b.kind) return a.kind === "link" ? -1 : 1; if (a.kind === "style" && b.kind === "style") return (STYLE_RANK.get(a.style) ?? 0) - (STYLE_RANK.get(b.style) ?? 0); return a.index - b.index; }); for (const item of openingItems) { out += item.open; stack.push({ close: item.close, end: item.end }); } } const next = points[i + 1]; if (next === void 0) break; if (next > pos) out += options.escapeText(text.slice(pos, next)); } return out; } //#endregion //#region packages/markdown-core/src/tables.ts const MARKDOWN_STYLE_MARKERS = { bold: { open: "**", close: "**" }, italic: { open: "_", close: "_" }, strikethrough: { open: "~~", close: "~~" }, code: { open: "`", close: "`" }, code_block: { open: "```\n", close: "```" } }; /** Converts markdown tables into the configured plaintext/code rendering mode. */ function convertMarkdownTables(markdown, mode) { if (!markdown || mode === "off") return markdown; const { ir, hasTables } = markdownToIRWithMeta(markdown, { linkify: false, autolink: false, headingStyle: "none", blockquotePrefix: "", tableMode: mode === "block" ? "code" : mode }); if (!hasTables) return markdown; return renderMarkdownWithMarkers(ir, { styleMarkers: MARKDOWN_STYLE_MARKERS, escapeText: (text) => text, buildLink: (link, text) => { const href = link.href.trim(); if (!href) return null; if (!text.slice(link.start, link.end)) return null; return { start: link.start, end: link.end, open: "[", close: `](${href})` }; } }); } //#endregion export { markdownToIRWithMeta as a, markdownToIR as i, renderMarkdownWithMarkers as n, sliceMarkdownIR as o, chunkMarkdownIR as r, convertMarkdownTables as t };