legal-markdown-js
Version:
Node.js implementation of LegalMarkdown for processing legal documents with markdown and YAML - Complete feature parity with Ruby version
555 lines • 21.3 kB
JavaScript
/**
* Remark Plugin for Import Processing
*
* This plugin processes import directives in legal documents using AST processing.
* Imports allow including content from external files, with support for partial
* content inclusion, metadata merging, and circular import detection.
*
* Features:
* - File-based imports with @import directive
* - Partial content imports from files
* - YAML frontmatter merging from imported files
* - Circular import detection and prevention
* - Relative and absolute path resolution
* - Import caching for performance
*
* @example
* ```typescript
* import { unified } from 'unified';
* import remarkParse from 'remark-parse';
* import remarkStringify from 'remark-stringify';
* import { remarkImports } from './imports.js';
*
* const processor = unified()
* .use(remarkParse)
* .use(remarkImports, {
* basePath: './documents',
* mergeMetadata: true
* })
* .use(remarkStringify);
* ```
*
* @module
*/
import { visit, SKIP } from 'unist-util-visit';
import * as fs from 'fs';
import * as path from 'path';
import { unified } from 'unified';
import remarkParse from 'remark-parse';
import { parseYamlFrontMatter } from '../../core/parsers/yaml-parser.js';
import { mergeSequentially } from '../../core/utils/frontmatter-merger.js';
import { ProcessingPhase } from './types.js';
import { logger } from '../../utils/logger.js';
/**
* Remark plugin for processing imports
*
* This plugin identifies and processes import directives in markdown text,
* loading content from external files and optionally merging their metadata.
*
* @param options - Configuration options for import processing
* @returns Remark plugin transformer function
*/
export const remarkImports = options => {
const { basePath = '.', mergeMetadata = true, debug = false, maxDepth = 10, timeoutMs = 30000, filterReserved = true, validateTypes = true, logImportOperations = false, importTracing = false, onMetadataMerged, importStack = [], } = options;
return async (tree) => {
const startTime = Date.now();
if (debug) {
logger.debug('Processing imports with options:', {
basePath,
mergeMetadata,
maxDepth,
timeoutMs,
filterReserved,
validateTypes,
currentDepth: importStack.length,
});
}
const context = {
depth: importStack.length,
maxDepth,
basePath,
mergeMetadata,
debug,
startTime,
timeoutMs,
filterReserved,
validateTypes,
logImportOperations,
importTracing,
onMetadataMerged,
importStack: importStack.map(stackPath => getCanonicalImportPath(stackPath)),
contentCache: new Map(),
importedMetadataList: [],
importedFiles: [],
accumulatedMetadata: {}, // Initialize with empty metadata
};
const replacements = [];
// Process all paragraph nodes that might contain import directives
visit(tree, 'paragraph', (node, index, parent) => {
if (!parent || index === undefined)
return;
// Check if any text children contain import directives
for (const child of node.children) {
if (child.type === 'text') {
const directives = extractImportDirectives(child.value);
if (directives.length > 0) {
// This paragraph contains imports - we'll need to process it
replacements.push({
parent: parent,
index,
nodes: [], // Will be filled in later
});
return SKIP; // Skip children
}
}
}
});
// Process replacements (parse imported content to AST)
for (const replacement of replacements) {
const paragraph = replacement.parent.children[replacement.index];
const processedNodes = await processParagraphWithImports(paragraph, context);
replacement.nodes = processedNodes;
}
// Apply replacements (in reverse order to maintain indices)
for (let i = replacements.length - 1; i >= 0; i--) {
const { parent, index, nodes } = replacements[i];
parent.children.splice(index, 1, ...nodes);
}
// Perform sequential merge of all imported metadata at the end
if (context.mergeMetadata && context.importedMetadataList.length > 0) {
const mergedResult = performSequentialMerge(context);
tree._importedMetadata = mergedResult.metadata;
tree._importStats = mergedResult.stats;
}
};
};
/**
* Process a paragraph containing import directives and return AST nodes
*
* This function replaces the old text-based import logic. Instead of replacing
* @import directives with plain text (which causes HTML escaping), it parses
* imported content into proper AST nodes.
*
* @param paragraph - Paragraph node containing import directives
* @param context - Import processing context
* @returns Array of Content nodes to replace the paragraph
*/
async function processParagraphWithImports(paragraph, context) {
const results = [];
// Process each child in the paragraph
for (const child of paragraph.children) {
if (child.type !== 'text') {
results.push(child);
continue;
}
const text = child.value;
const directives = extractImportDirectives(text);
if (directives.length === 0) {
// No imports, keep the text as is
results.push(child);
continue;
}
if (context.debug) {
logger.debug(`Found ${directives.length} import directives`);
}
// Process each import directive and build AST nodes
let lastIndex = 0;
for (const directive of directives) {
// Add text before the import directive (if any)
if (directive.start > lastIndex) {
const beforeText = text.substring(lastIndex, directive.start);
if (beforeText.trim()) {
results.push({
type: 'text',
value: beforeText,
});
}
}
// Process the import and get AST nodes
const importedNodes = await processImportDirectiveToAST(directive, context);
results.push(...importedNodes);
lastIndex = directive.end;
}
// Add remaining text after last import (if any)
if (lastIndex < text.length) {
const afterText = text.substring(lastIndex);
if (afterText.trim()) {
results.push({
type: 'text',
value: afterText,
});
}
}
}
return results;
}
/**
* Extract import directives from text
*/
function extractImportDirectives(text) {
const directives = [];
// Regex to match import directives:
// @import file.md (unquoted)
// @import "path/to file.md" (double-quoted, allows spaces)
// @import file.md#section (with section anchor)
const importRegex = /@import\s+(?:"([^"]+)"|([^\s#]+))(?:#([^\s]+))?/g;
let match;
while ((match = importRegex.exec(text)) !== null) {
const fullMatch = match[0];
// Group 1 = quoted path, Group 2 = unquoted path, Group 3 = section
const filePath = (match[1] || match[2]).trim();
const section = match[3]?.trim();
directives.push({
filePath,
section,
start: match.index,
end: match.index + fullMatch.length,
fullMatch,
});
}
return directives;
}
/**
* Parse imported content into AST nodes
*
* This function parses markdown content into proper AST nodes instead of plain text,
* which preserves HTML structure and prevents HTML comments/tags from being escaped.
*
* @param content - Markdown content to parse
* @returns Array of AST nodes (Content[])
*/
async function parseImportedContentToAST(content) {
// Create a minimal unified processor with just remark-parse
const processor = unified().use(remarkParse);
// Parse the content to get AST
const tree = processor.parse(content);
// Return the children nodes (not the root itself)
// This allows us to insert them into the parent tree
return tree.children;
}
/**
* Process an import directive and return AST nodes
*
* This replaces the old processImportDirective() which returned plain text.
* By returning AST nodes, we preserve HTML structure and prevent escaping.
*
* @param directive - Import directive information
* @param context - Import processing context
* @returns Array of AST nodes from the imported content
*/
async function processImportDirectiveToAST(directive, context) {
if (context.depth >= context.maxDepth) {
logger.warn(`Maximum import depth (${context.maxDepth}) reached for file "${directive.filePath}"`);
// Return the original directive as text
return [{ type: 'text', value: directive.fullMatch }];
}
// Resolve file path
const absolutePath = path.resolve(context.basePath, directive.filePath);
const normalizedPath = path.normalize(absolutePath);
const canonicalPath = getCanonicalImportPath(normalizedPath);
// Check for circular imports
if (context.importStack.includes(canonicalPath)) {
logger.warn(`Circular import detected: ${canonicalPath}`);
return [{ type: 'text', value: directive.fullMatch }];
}
if (context.debug) {
logger.debug(`Processing import "${directive.filePath}" (resolved: ${normalizedPath})`);
}
// Load file content
const fileContent = loadFileContent(canonicalPath, context);
if (!fileContent) {
logger.warn(`Import file not found: ${directive.filePath}`);
return [{ type: 'text', value: directive.fullMatch }];
}
// Parse YAML frontmatter if mergeMetadata is enabled
let contentToImport = fileContent;
if (context.mergeMetadata) {
const { content, metadata } = parseYamlFrontMatter(fileContent, false);
contentToImport = content;
if (Object.keys(metadata).length > 0) {
// Add metadata to list for sequential merging later
context.importedMetadataList.push({
metadata,
source: canonicalPath,
});
// Accumulate metadata for mixin expansion in imported content
// Merge the new metadata into accumulated metadata (shallow merge)
context.accumulatedMetadata = {
...context.accumulatedMetadata,
...metadata,
};
// Track imported file
if (!context.importedFiles.includes(canonicalPath)) {
context.importedFiles.push(canonicalPath);
}
if (context.debug) {
logger.debug(`Collected metadata from ${directive.filePath}:`, Object.keys(metadata));
}
}
}
// Extract section if specified
if (directive.section) {
contentToImport = extractSection(contentToImport, directive.section, context.debug);
}
// Process nested imports in the content
const nestedContext = {
...context,
depth: context.depth + 1,
importStack: [...context.importStack, canonicalPath],
basePath: path.dirname(canonicalPath), // Update base path for relative imports
};
const processedContent = await processNestedImportsToAST(contentToImport, nestedContext);
// Expand mixins using accumulated metadata
// This allows mixins defined in imported file frontmatter to work
let result;
if (context.mergeMetadata && Object.keys(context.accumulatedMetadata).length > 0) {
result = await expandMixinsInAST(processedContent, context.accumulatedMetadata);
}
else {
result = processedContent;
}
// Wrap with HTML comment nodes if importTracing is enabled
if (context.importTracing) {
const relPath = path.relative(context.basePath, canonicalPath);
const startNode = {
type: 'html',
value: `<!-- start import: ${relPath} -->`,
};
const endNode = {
type: 'html',
value: `<!-- end import: ${relPath} -->`,
};
return [startNode, ...result, endNode];
}
return result;
}
/**
* Perform sequential merge of all collected metadata
*/
function performSequentialMerge(context) {
if (context.importedMetadataList.length === 0) {
return {
metadata: {},
stats: undefined,
};
}
// Check timeout before starting merge
if (Date.now() - context.startTime > context.timeoutMs) {
throw new Error(`Import processing timed out after ${context.timeoutMs}ms. ` +
'This may indicate complex nested imports or slow file operations.');
}
// Prepare merge options
const mergeOptions = {
filterReserved: context.filterReserved,
validateTypes: context.validateTypes,
logOperations: context.logImportOperations,
includeStats: true,
timeoutMs: Math.max(1000, context.timeoutMs - (Date.now() - context.startTime)),
};
// Extract metadata array for merging
const metadataList = context.importedMetadataList.map(item => item.metadata);
if (context.debug) {
logger.debug(`Performing sequential merge of ${metadataList.length} metadata objects`);
}
try {
// Use sequential merge with initial empty metadata (source always wins)
const result = mergeSequentially({}, metadataList, mergeOptions);
// Call onMetadataMerged callback if provided
if (context.onMetadataMerged && Object.keys(result.metadata).length > 0) {
context.onMetadataMerged(result.metadata, 'merged-imports');
}
return result;
}
catch (error) {
if (context.debug) {
logger.warn(`Sequential merge failed: ${error instanceof Error ? error.message : String(error)}`);
}
throw error;
}
}
/**
* Load file content with caching
*/
function loadFileContent(filePath, context) {
// Check cache first
if (context.contentCache.has(filePath)) {
return context.contentCache.get(filePath);
}
try {
if (fs.existsSync(filePath)) {
const content = fs.readFileSync(filePath, 'utf-8');
context.contentCache.set(filePath, content);
return content;
}
}
catch (error) {
if (context.debug) {
logger.warn(`Failed to load import file "${filePath}": ${error instanceof Error ? error.message : String(error)}`);
}
}
return null;
}
function getCanonicalImportPath(filePath) {
const normalizedPath = path.normalize(filePath);
try {
const realPath = fs.realpathSync.native
? fs.realpathSync.native(normalizedPath)
: fs.realpathSync(normalizedPath);
return path.normalize(realPath);
}
catch {
return normalizedPath;
}
}
/**
* Extract a specific section from content
*/
function extractSection(content, sectionName, debug) {
// Look for header with the section name
const lines = content.split('\n');
let sectionStart = -1;
let sectionEnd = lines.length;
let sectionLevel = 0;
// Find section start
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
// Check if this is a header line
const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
if (headerMatch) {
const [, hashes, title] = headerMatch;
const level = hashes.length;
if (title.toLowerCase() === sectionName.toLowerCase()) {
sectionStart = i + 1; // Start after the header
sectionLevel = level;
if (debug) {
logger.debug(`Found section '${sectionName}' at line ${i}`);
}
break;
}
}
}
if (sectionStart === -1) {
if (debug) {
logger.debug(`Section '${sectionName}' not found`);
}
return content; // Return full content if section not found
}
// Find section end (next header of same or higher level)
for (let i = sectionStart; i < lines.length; i++) {
const line = lines[i].trim();
const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
if (headerMatch) {
const [, hashes] = headerMatch;
const level = hashes.length;
if (level <= sectionLevel) {
sectionEnd = i;
break;
}
}
}
return lines.slice(sectionStart, sectionEnd).join('\n').trim();
}
// expandMixinsWithTracking() removed; see PR for details.
/**
* Process nested imports in content and return AST nodes
*
* This replaces the old processNestedImports() which worked with text.
* By returning AST nodes, HTML comments and tags are preserved properly.
*
* @param content - Markdown content that may contain import directives
* @param context - Import processing context
* @returns Array of AST nodes with imports resolved
*/
async function processNestedImportsToAST(content, context) {
// Parse the content to AST first
const nodes = await parseImportedContentToAST(content);
// Check if any of the nodes contain import directives
const hasImports = nodes.some(node => {
if (node.type === 'paragraph') {
return node.children.some(child => child.type === 'text' && extractImportDirectives(child.value).length > 0);
}
return false;
});
if (!hasImports) {
// No nested imports, return as is
return nodes;
}
// Process paragraphs with imports
const processedNodes = [];
for (const node of nodes) {
if (node.type === 'paragraph') {
const paragraph = node;
const hasDirectives = paragraph.children.some(child => child.type === 'text' && extractImportDirectives(child.value).length > 0);
if (hasDirectives) {
// Process this paragraph's imports
const importedNodes = await processParagraphWithImports(paragraph, context);
processedNodes.push(...importedNodes);
}
else {
processedNodes.push(node);
}
}
else {
processedNodes.push(node);
}
}
return processedNodes;
}
/**
* Expand mixins in AST nodes
*
* This function processes {{field}} patterns in text nodes within the AST,
* resolving them using the provided metadata and wrapping them with HTML
* for field tracking.
*
* @param nodes - AST nodes that may contain mixin patterns
* @param metadata - Metadata to resolve field values
* @returns AST nodes with mixins expanded
*/
async function expandMixinsInAST(nodes, _metadata) {
const processedNodes = [];
for (const node of nodes) {
if (node.type === 'paragraph') {
// Process text children in the paragraph
const paragraph = node;
const processedChildren = [];
for (const child of paragraph.children) {
if (child.type === 'text') {
// Let remarkTemplateFields handle all mixin processing with proper metadata
processedChildren.push(child);
}
else {
processedChildren.push(child);
}
}
processedNodes.push({
...paragraph,
children: processedChildren,
});
}
else {
processedNodes.push(node);
}
}
return processedNodes;
}
/**
* Metadata for remarkImports plugin
*
* Dependencies:
* - Must run BEFORE remarkTemplateFields (fields in imported content need processing)
* - Must run BEFORE remarkLegalHeadersParser (legal headers in imported content need parsing)
* - Must run BEFORE remarkFieldTracking (imported fields need tracking)
*/
// Exported for testing - not part of public API
export { extractImportDirectives as _extractImportDirectives, loadFileContent as _loadFileContent, getCanonicalImportPath as _getCanonicalImportPath, extractSection as _extractSection, };
const _remarkImportsMetadata = {
name: 'remarkImports',
phase: ProcessingPhase.CONTENT_LOADING,
description: 'Process @import directives and insert content as AST nodes',
capabilities: ['content:imported', 'metadata:merged'],
runBefore: ['remarkTemplateFields', 'remarkLegalHeadersParser', 'remarkFieldTracking'],
required: false,
version: '2.0.0', // Version 2.0 uses AST-based insertion
};
//# sourceMappingURL=imports.js.map