roam-research-mcp
Version:
MCP server and CLI for Roam Research
540 lines (539 loc) • 22.5 kB
JavaScript
import { randomBytes } from 'crypto';
// Regex patterns for markdown elements
const NUMBERED_LIST_REGEX = /^(\s*)\d+\.\s+(.*)$/;
const HORIZONTAL_RULE_REGEX = /^(\s*)(-{3,}|\*{3,}|_{3,})\s*$/;
/**
* Check if text has a traditional markdown table
*/
function hasMarkdownTable(text) {
return /^\|([^|]+\|)+\s*$\n\|(\s*:?-+:?\s*\|)+\s*$\n(\|([^|]+\|)+\s*$\n*)+$/.test(text);
}
/**
* Converts a markdown table to Roam format
*/
function convertTableToRoamFormat(text) {
const lines = text.split('\n')
.map(line => line.trim())
.filter(line => line.length > 0);
const tableRegex = /^\|([^|]+\|)+\s*$\n\|(\s*:?-+:?\s*\|)+\s*$\n(\|([^|]+\|)+\s*$\n*)+/m;
if (!tableRegex.test(text)) {
return text;
}
const rows = lines
.filter((_, index) => index !== 1)
.map(line => line.trim()
.replace(/^\||\|$/g, '')
.split('|')
.map(cell => cell.trim()));
let roamTable = '{{[[table]]}}\n';
// First row becomes column headers
const headers = rows[0];
for (let i = 0; i < headers.length; i++) {
roamTable += `${' '.repeat(i + 1)}- ${headers[i]}\n`;
}
// Remaining rows become nested under each column
for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
const row = rows[rowIndex];
for (let colIndex = 0; colIndex < row.length; colIndex++) {
roamTable += `${' '.repeat(colIndex + 1)}- ${row[colIndex]}\n`;
}
}
return roamTable.trim();
}
function convertAllTables(text) {
return text.replaceAll(/(^\|([^|]+\|)+\s*$\n\|(\s*:?-+:?\s*\|)+\s*$\n(\|([^|]+\|)+\s*$\n*)+)/gm, (match) => {
return '\n' + convertTableToRoamFormat(match) + '\n';
});
}
/**
* Parse markdown heading syntax (e.g. "### Heading") and return the heading level (1-3) and content.
* Heading level is determined by the number of # characters (e.g. # = h1, ## = h2, ### = h3).
* Returns heading_level: 0 for non-heading content.
*/
function parseMarkdownHeadingLevel(text) {
const match = text.match(/^(#{1,3})\s+(.+)$/);
if (match) {
return {
heading_level: match[1].length, // Number of # characters determines heading level
content: match[2].trim()
};
}
return {
heading_level: 0, // Not a heading
content: text.trim()
};
}
function convertToRoamMarkdown(text) {
// Protect inline code and code blocks from transformation
const codeBlocks = [];
// Use null bytes to create a unique placeholder that won't be transformed
const PLACEHOLDER_START = '\x00\x01CB';
const PLACEHOLDER_END = '\x02\x00';
// Extract code blocks (``` ... ```) first
text = text.replace(/```[\s\S]*?```/g, (match) => {
codeBlocks.push(match);
return `${PLACEHOLDER_START}${codeBlocks.length - 1}${PLACEHOLDER_END}`;
});
// Extract inline code (` ... `)
text = text.replace(/`[^`]+`/g, (match) => {
codeBlocks.push(match);
return `${PLACEHOLDER_START}${codeBlocks.length - 1}${PLACEHOLDER_END}`;
});
// Handle double asterisks/underscores (bold)
text = text.replace(/\*\*(.+?)\*\*/g, '**$1**'); // Preserve double asterisks
// Handle single asterisks/underscores (italic)
text = text.replace(/(?<!\*)\*(?!\*)(.+?)(?<!\*)\*(?!\*)/g, '__$1__'); // Single asterisk to double underscore
// Single underscore to italic, but only at word boundaries. Underscores flanked by
// word chars (\w includes _) are literal per CommonMark, so snake_case filenames and
// URLs like /wiki/Ning_Li_(physicist) are left intact instead of becoming __Li__.
text = text.replace(/(?<!\w)_(?!_)(.+?)_(?!\w)/g, '__$1__');
// Handle highlights
text = text.replace(/==(.+?)==/g, '^^$1^^');
// Convert tasks
text = text.replace(/- \[ \]/g, '- {{[[TODO]]}}');
text = text.replace(/- \[x\]/g, '- {{[[DONE]]}}');
// Convert tables
text = convertAllTables(text);
// Restore protected code blocks
text = text.replace(new RegExp(`${PLACEHOLDER_START}(\\d+)${PLACEHOLDER_END}`, 'g'), (_, index) => {
return codeBlocks[parseInt(index, 10)];
});
return text;
}
/**
* Is this line a bullet whose only content is a code-fence opener?
*
* Only such a line may be spliced into "bullet" + "fence" so the fence state
* machine can gather the following lines. Any line with content AFTER the
* fence is a block that merely CONTAINS backticks — splicing it opens a region
* that never closes, and the parser then consumes the rest of the document.
*
* That was a real, unrecoverable defect: `- wrap it in ``` to make code`
* followed by three blocks parsed to a single block "wrap it in", and
* roam_update_page_markdown deleted the other three. Roam has no undo.
*
* This subsumes the earlier balanced-fence guard: a line carrying both an
* opening and a closing fence necessarily has content after the opener, so it
* never matches this shape.
*/
function isBulletFenceOpener(trimmedLine) {
return /^\s*[-*+]\s+```[A-Za-z0-9_+-]*\s*$/.test(trimmedLine);
}
/**
* A fence line with content after its opening ``` is content, not a region
* opener. Guards the bare (non-bullet) case that the splice rule cannot see,
* e.g. a line rendered as ```js\ncode\n``` with no bullet prefix.
*/
function fenceHasTrailingContent(trimmedLine) {
const open = trimmedLine.indexOf('```');
if (open === -1)
return false;
return trimmedLine.slice(open + 3).replace(/^[A-Za-z0-9_+-]*/, '').trim().length > 0;
}
function parseMarkdown(markdown) {
markdown = convertToRoamMarkdown(markdown);
const originalLines = markdown.split('\n');
const processedLines = [];
// Pre-process lines to handle mid-line code blocks without splice
for (const line of originalLines) {
const trimmedLine = line.trimEnd();
const codeStartIndex = trimmedLine.indexOf('```');
if (codeStartIndex > 0 && isBulletFenceOpener(trimmedLine)) {
// Under this rule the text before the fence is ALWAYS just the bullet
// marker (isBulletFenceOpener only matches "bullet + nothing but the
// fence"), so there is no real content to preserve as its own node.
// Pushing it anyway used to leave a bare "-" line that the parser can't
// recognise as a bullet once trimmed (no trailing content survives
// trimEnd), so it fell through to the plain-line branch and emitted a
// spurious "-" block ahead of the code block it introduces. Dropping it
// loses nothing: the fence line below carries the same leading
// whitespace, so indentation-based nesting is unaffected.
const indentationWhitespace = line.match(/^\s*/)?.[0] ?? '';
processedLines.push(indentationWhitespace + trimmedLine.substring(codeStartIndex));
}
else {
processedLines.push(line);
}
}
// First pass: collect all unique indentation values to build level mapping
const indentationSet = new Set();
indentationSet.add(0); // Always include level 0
let inCodeBlockFirstPass = false;
for (const line of processedLines) {
const trimmedLine = line.trimEnd();
if (trimmedLine.match(/^(\s*)```/) && !fenceHasTrailingContent(trimmedLine)) {
inCodeBlockFirstPass = !inCodeBlockFirstPass;
if (!inCodeBlockFirstPass)
continue; // Skip closing ```
const indent = line.match(/^\s*/)?.[0].length ?? 0;
indentationSet.add(indent);
continue;
}
if (inCodeBlockFirstPass || trimmedLine === '')
continue;
// Check for numbered list, bullet list, or plain line
const numberedMatch = line.match(NUMBERED_LIST_REGEX);
const bulletMatch = trimmedLine.match(/^(\s*)[-*+]\s+/);
if (numberedMatch) {
indentationSet.add(numberedMatch[1].length);
}
else if (bulletMatch) {
indentationSet.add(bulletMatch[1].length);
}
else {
const indent = line.match(/^\s*/)?.[0].length ?? 0;
indentationSet.add(indent);
}
}
// Create sorted array of indentation values and map to sequential levels
const sortedIndents = Array.from(indentationSet).sort((a, b) => a - b);
const indentToLevel = new Map();
sortedIndents.forEach((indent, index) => {
indentToLevel.set(indent, index);
});
// Helper to get level from indentation, finding closest match
function getLevel(indent) {
if (indentToLevel.has(indent)) {
return indentToLevel.get(indent);
}
// Find the closest smaller indentation
let closestLevel = 0;
for (const [ind, lvl] of indentToLevel) {
if (ind <= indent && lvl > closestLevel) {
closestLevel = lvl;
}
}
return closestLevel;
}
const rootNodes = [];
const stack = [];
let inCodeBlock = false;
let codeBlockContent = '';
let codeBlockIndentation = 0;
let codeBlockParentLevel = 0;
for (let i = 0; i < processedLines.length; i++) {
const line = processedLines[i];
const trimmedLine = line.trimEnd();
if (trimmedLine.match(/^(\s*)```/) && !fenceHasTrailingContent(trimmedLine)) {
if (!inCodeBlock) {
inCodeBlock = true;
codeBlockContent = trimmedLine.trimStart() + '\n';
codeBlockIndentation = line.match(/^\s*/)?.[0].length ?? 0;
codeBlockParentLevel = stack.length;
}
else {
inCodeBlock = false;
codeBlockContent += trimmedLine.trimStart();
const linesInCodeBlock = codeBlockContent.split('\n');
let baseIndentation = '';
for (let j = 1; j < linesInCodeBlock.length - 1; j++) {
const codeLine = linesInCodeBlock[j];
if (codeLine.trim().length > 0) {
const indentMatch = codeLine.match(/^[\t ]*/);
if (indentMatch) {
baseIndentation = indentMatch[0];
break;
}
}
}
const processedCodeLines = linesInCodeBlock.map((codeLine, index) => {
if (index === 0 || index === linesInCodeBlock.length - 1)
return codeLine.trimStart();
if (codeLine.trim().length === 0)
return '';
if (codeLine.startsWith(baseIndentation)) {
return codeLine.slice(baseIndentation.length);
}
return codeLine.trimStart();
});
const level = getLevel(codeBlockIndentation);
const node = {
content: processedCodeLines.join('\n'),
level,
children: []
};
while (stack.length > codeBlockParentLevel) {
stack.pop();
}
if (level === 0) {
rootNodes.push(node);
stack[0] = node;
}
else {
while (stack.length > level) {
stack.pop();
}
if (stack[level - 1]) {
stack[level - 1].children.push(node);
}
else {
rootNodes.push(node);
}
stack[level] = node;
}
codeBlockContent = '';
}
continue;
}
if (inCodeBlock) {
codeBlockContent += line + '\n';
continue;
}
if (trimmedLine === '') {
continue;
}
// Check for horizontal rule (---, ***, ___)
const hrMatch = line.match(HORIZONTAL_RULE_REGEX);
if (hrMatch) {
const hrIndentation = hrMatch[1].length;
const hrLevel = getLevel(hrIndentation);
const hrNode = {
content: '---', // Roam's HR format
level: hrLevel,
is_hr: true,
children: []
};
while (stack.length > hrLevel) {
stack.pop();
}
if (hrLevel === 0 || !stack[hrLevel - 1]) {
rootNodes.push(hrNode);
stack[0] = hrNode;
}
else {
stack[hrLevel - 1].children.push(hrNode);
}
stack[hrLevel] = hrNode;
continue;
}
let indentation;
let contentToParse;
let isNumberedItem = false;
// Check for numbered list item (1., 2., etc.)
const numberedMatch = line.match(NUMBERED_LIST_REGEX);
const bulletMatch = trimmedLine.match(/^(\s*)[-*+]\s+/);
if (numberedMatch) {
indentation = numberedMatch[1].length;
contentToParse = numberedMatch[2];
isNumberedItem = true;
}
else if (bulletMatch) {
indentation = bulletMatch[1].length;
contentToParse = trimmedLine.substring(bulletMatch[0].length);
}
else {
indentation = line.match(/^\s*/)?.[0].length ?? 0;
contentToParse = trimmedLine;
}
const level = getLevel(indentation);
const { heading_level, content: finalContent } = parseMarkdownHeadingLevel(contentToParse);
const node = {
content: finalContent,
level,
...(heading_level > 0 && { heading_level }),
children: []
};
while (stack.length > level) {
stack.pop();
}
if (level === 0 || !stack[level - 1]) {
rootNodes.push(node);
stack[0] = node;
// Root-level numbered items: no parent to set view type on
// They'll appear as regular blocks (Roam doesn't support numbered view at root)
}
else {
const parent = stack[level - 1];
parent.children.push(node);
// If this is the first numbered item under a parent, set parent's view type
if (isNumberedItem && parent.children_view_type !== 'numbered') {
parent.children_view_type = 'numbered';
}
}
stack[level] = node;
}
return rootNodes;
}
function parseTableRows(lines) {
const tableNodes = [];
let currentLevel = -1;
for (const line of lines) {
const trimmedLine = line.trimEnd();
if (!trimmedLine)
continue;
// Calculate indentation level
const indentation = line.match(/^\s*/)?.[0].length ?? 0;
const level = Math.floor(indentation / 2);
// Extract content after bullet point
const content = trimmedLine.replace(/^\s*[-*+]\s*/, '');
// Create node for this cell
const node = {
content,
level,
children: []
};
// Track the first level we see to maintain relative nesting
if (currentLevel === -1) {
currentLevel = level;
}
// Add node to appropriate parent based on level
if (level === currentLevel) {
tableNodes.push(node);
}
else {
// Find parent by walking back through nodes
let parent = tableNodes[tableNodes.length - 1];
while (parent && parent.level < level - 1) {
parent = parent.children[parent.children.length - 1];
}
if (parent) {
parent.children.push(node);
}
}
}
return tableNodes;
}
export function generateBlockUid() {
// Generate a random string of 9 characters (Roam's format) using crypto for better randomness
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_';
// 64 chars, which divides 256 evenly (256 = 64 * 4), so simple modulo is unbiased
const bytes = randomBytes(9);
let uid = '';
for (let i = 0; i < 9; i++) {
uid += chars[bytes[i] % 64];
}
return uid;
}
/**
* Reorganise a parsed markdown node list into a "section tree" based on heading
* levels: each heading (e.g. ##) becomes a parent, and the blocks that follow it
* nest under it until a heading of equal-or-higher level closes the section;
* deeper headings (###) nest further. Blocks before the first heading — and
* inputs with no headings at all — are returned unchanged, so indentation-only
* markdown and intentional flat lists are unaffected.
*
* This fixes flush-left, heading-structured markdown (e.g. report skeletons)
* that would otherwise import as a flat list of root-level siblings, because
* neither indentation-based nesting nor explicit levels capture the document
* outline encoded in the heading syntax.
*/
function nestUnderHeadings(nodes) {
const result = [];
const stack = []; // currently-open heading nodes, shallow → deep
for (const original of nodes) {
// Recurse first so headings appearing inside indented structure nest too
const node = { ...original, children: nestUnderHeadings(original.children) };
const level = node.heading_level ?? 0;
if (level > 0) {
// A heading at the same or shallower level closes open equal/deeper sections
while (stack.length > 0 && (stack[stack.length - 1].heading_level ?? 0) >= level) {
stack.pop();
}
}
// Attach under the deepest open heading, or at root when no section is open
(stack.length > 0 ? stack[stack.length - 1].children : result).push(node);
if (level > 0) {
stack.push(node);
}
}
return result;
}
function convertNodesToBlocks(nodes) {
return nodes.map(node => ({
uid: generateBlockUid(),
content: node.content,
...(node.heading_level && { heading_level: node.heading_level }), // Preserve heading level if present
...(node.children_view_type && { children_view_type: node.children_view_type }), // Preserve view type for numbered lists
children: convertNodesToBlocks(node.children)
}));
}
function buildActionsFromBlocks(blocks, parentUid, order = 'last') {
const actions = [];
// Helper function to recursively create actions
function createBlockActions(blocks, parentUid, order) {
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
// Create the current block
const action = {
action: 'create-block',
location: {
'parent-uid': parentUid,
order: typeof order === 'number' ? order + i : order
},
block: {
uid: block.uid,
string: block.content,
...(block.heading_level && { heading: block.heading_level }),
...(block.children_view_type && { 'children-view-type': block.children_view_type })
}
};
actions.push(action);
// Create child blocks if any
if (block.children.length > 0) {
createBlockActions(block.children, block.uid, 'last');
}
}
}
// Create all block actions
createBlockActions(blocks, parentUid, order);
return actions;
}
function convertToRoamActions(nodes, parentUid, order = 'last') {
// First convert nodes to blocks with UIDs
return buildActionsFromBlocks(convertNodesToBlocks(nodes), parentUid, order);
}
/**
* Like convertToRoamActions, but also returns the minted block tree (with the
* same client-generated UIDs used in the actions). Lets callers report exactly
* what was created WITHOUT a post-write re-query — Roam's write API does not
* echo back transacted UIDs, and we already know them because we minted them.
*/
function convertToRoamActionsWithBlocks(nodes, parentUid, order = 'last') {
const blocks = convertNodesToBlocks(nodes);
return { actions: buildActionsFromBlocks(blocks, parentUid, order), blocks };
}
/**
* Converts markdown nodes to Roam batch actions, grouped by nesting level.
* This ensures parent blocks exist before child blocks are created.
* Returns an array of action arrays, where index 0 contains root-level actions,
* index 1 contains first-level child actions, etc.
*/
function convertToRoamActionsStaged(nodes, parentUid, order = 'last') {
// First convert nodes to blocks with UIDs
const blocks = convertNodesToBlocks(nodes);
const actionsByLevel = [];
// Helper function to recursively create actions, tracking depth
function createBlockActions(blocks, parentUid, order, depth) {
// Ensure array exists for this depth
if (!actionsByLevel[depth]) {
actionsByLevel[depth] = [];
}
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
// Create the current block
const action = {
action: 'create-block',
location: {
'parent-uid': parentUid,
order: typeof order === 'number' ? order + i : order
},
block: {
uid: block.uid,
string: block.content,
...(block.heading_level && { heading: block.heading_level }),
...(block.children_view_type && { 'children-view-type': block.children_view_type })
}
};
actionsByLevel[depth].push(action);
// Create child blocks if any
if (block.children.length > 0) {
createBlockActions(block.children, block.uid, 'last', depth + 1);
}
}
}
// Create all block actions starting at depth 0
createBlockActions(blocks, parentUid, order, 0);
return actionsByLevel;
}
// Export public functions and types
export { parseMarkdown, nestUnderHeadings, convertToRoamActions, convertToRoamActionsWithBlocks, convertToRoamActionsStaged, hasMarkdownTable, convertAllTables, convertToRoamMarkdown, parseMarkdownHeadingLevel };