oneie/one/tools/lib/template-processor.js

🤝 ONE Personal Collaborative Intelligence - Creates personalized AI workspace from your me.md profile. Simple: npx oneie → edit me.md → generate personalized agents, workflows & missions. From students to enterprises, ONE adapts to your context.

/**
 * Template Processor for Apple-Style Book Generation
 * Handles hierarchical book structure detection, image mapping, and content processing
 * Supports 5-part structure: Foundation → ATTRACT → CONVERT → GROW → REFINE + Appendix
 */

import { promises as fs, accessSync } from 'fs';
import path from 'path';

/**
 * Strip YAML frontmatter from markdown content
 * @param {string} content - Markdown content with potential frontmatter
 * @returns {string} Content without frontmatter
 */
function stripYamlFrontmatter(content) {
  // Remove YAML frontmatter (--- ... --- at start of file)
  const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n/;
  content = content.replace(frontmatterRegex, '');
  
  // Also remove any remaining --- ... --- blocks in the content
  const inlineYamlRegex = /\n---\s*\n([\s\S]*?)\n---?\s*\n/g;
  content = content.replace(inlineYamlRegex, '\n\n');
  
  return content.trim();
}

/**
 * Detect hierarchical book structure from content directory
 * Supports folder-based structure: 0-introduction/, 1-foundation/, 2-attract/, etc.
 * @param {string} contentDir - Path to content directory
 * @param {string} imagesDir - Path to images directory
 * @returns {Object} Detected hierarchical book structure
 */
async function detectHierarchicalBookStructure(contentDir, imagesDir) {
  const structure = {
    introduction: { files: [], title: 'Introduction' },
    parts: [
      { number: 1, title: 'Foundation', chapters: [], folder: '' },
      { number: 2, title: 'ATTRACT', chapters: [], folder: '' },
      { number: 3, title: 'CONVERT', chapters: [], folder: '' },
      { number: 4, title: 'GROW', chapters: [], folder: '' },
      { number: 5, title: 'REFINE', chapters: [], folder: '' }
    ],
    appendix: { files: [], title: 'Appendix' },
    images: {}
  };

  try {
    const entries = await fs.readdir(contentDir, { withFileTypes: true });
    
    for (const entry of entries) {
      if (entry.isDirectory()) {
        const folderName = entry.name;
        const folderPath = path.join(contentDir, folderName);
        
        if (folderName.match(/^0-introduction/)) {
          // Introduction folder
          const introFiles = await scanFolderForFiles(folderPath, imagesDir);
          structure.introduction.files = introFiles;
        } else if (folderName.match(/^[1-5]-/)) {
          // Part folders (1-foundation, 2-attract, etc.)
          const partMatch = folderName.match(/^(\d+)-(.+)$/);
          if (partMatch) {
            const [, partNum, partName] = partMatch;
            const partIndex = parseInt(partNum) - 1;
            
            if (partIndex >= 0 && partIndex < structure.parts.length) {
              const partFiles = await scanFolderForFiles(folderPath, imagesDir);
              structure.parts[partIndex].title = partName.toUpperCase();
              structure.parts[partIndex].folder = folderName;
              structure.parts[partIndex].chapters = await parseHierarchicalFiles(partFiles, folderName);
            }
          }
        } else if (folderName === 'appendix') {
          // Appendix folder
          const appendixFiles = await scanFolderForFiles(folderPath, imagesDir);
          structure.appendix.files = appendixFiles;
        }
      }
    }
    
    // Sort all parts and their chapters
    structure.parts.forEach(part => {
      part.chapters.sort((a, b) => {
        // Sort by hierarchical numbering
        const aSort = `${a.chapterNumber || 0}.${a.sectionNumber || 0}.${a.subSectionNumber || 0}`;
        const bSort = `${b.chapterNumber || 0}.${b.sectionNumber || 0}.${b.subSectionNumber || 0}`;
        return aSort.localeCompare(bSort, undefined, { numeric: true });
      });
      
      // Sort sections within each chapter
      part.chapters.forEach(chapter => {
        if (chapter.sections) {
          chapter.sections.sort((a, b) => {
            const aSort = `${a.sectionNumber || 0}.${a.subSectionNumber || 0}`;
            const bSort = `${b.sectionNumber || 0}.${b.subSectionNumber || 0}`;
            return aSort.localeCompare(bSort, undefined, { numeric: true });
          });
        }
      });
    });
    
    // Sort introduction and appendix files
    structure.introduction.files.sort((a, b) => parseFloat(a.number) - parseFloat(b.number));
    structure.appendix.files.sort((a, b) => a.number.localeCompare(b.number));
    
    return structure;
    
  } catch (error) {
    console.error('Error detecting hierarchical book structure:', error);
    throw error;
  }
}

/**
 * Scan a folder for markdown files and return file information
 * @param {string} folderPath - Path to folder
 * @param {string} imagesDir - Images directory path
 * @returns {Array} Array of file information objects
 */
async function scanFolderForFiles(folderPath, imagesDir) {
  const files = [];
  
  try {
    const entries = await fs.readdir(folderPath);
    
    for (const entry of entries) {
      if (entry.endsWith('.md')) {
        const filePath = path.join(folderPath, entry);
        const fileInfo = await parseHierarchicalFile(entry, folderPath, imagesDir);
        if (fileInfo) {
          files.push(fileInfo);
        }
      }
    }
  } catch (error) {
    console.error(`Error scanning folder ${folderPath}:`, error);
  }
  
  return files;
}

/**
 * Parse hierarchical files within a folder
 * @param {Array} files - Array of file information objects
 * @param {string} folderName - Name of the folder
 * @returns {Array} Array of parsed chapter/section objects
 */
async function parseHierarchicalFiles(files, folderName) {
  const chapters = [];
  
  for (const file of files) {
    const fileName = file.file;
    
    // Parse files like: 1-foundation.md, 1.1-company.md, 1.1.2-tokenomics.md
    const match = fileName.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?-(.+)\.md$/);
    if (match) {
      const [, chapterNum, sectionNum, subSectionNum, name] = match;
      
      const chapterInfo = {
        file: fileName,
        folder: folderName,
        chapterNumber: parseInt(chapterNum),
        sectionNumber: sectionNum ? parseInt(sectionNum) : null,
        subSectionNumber: subSectionNum ? parseInt(subSectionNum) : null,
        name: name,
        title: formatTitle(name),
        number: fileName.match(/^(\d+(?:\.\d+)*)/)[1],
        level: sectionNum ? (subSectionNum ? 3 : 2) : 1,
        sections: [],
        fullPageImage: false,
        imageName: null
      };
      
      // Check for associated image
      await checkForImage(chapterInfo, file.imagesDir);
      
      chapters.push(chapterInfo);
    }
  }
  
  return chapters;
}

/**
 * Parse hierarchical file information
 * @param {string} file - Filename
 * @param {string} folderPath - Path to folder containing the file
 * @param {string} imagesDir - Images directory path
 * @returns {Object|null} Parsed file information
 */
async function parseHierarchicalFile(file, folderPath, imagesDir) {
  const fileInfo = {
    file,
    folderPath,
    imagesDir,
    fullPageImage: false,
    imageName: null
  };
  
  return fileInfo;
}

/**
 * Format title from filename
 * @param {string} name - Name from filename
 * @returns {string} Formatted title
 */
function formatTitle(name) {
  return name.split('-').map(word => 
    word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()
  ).join(' ');
}

/**
 * Check for associated image for a chapter/section
 * @param {Object} chapterInfo - Chapter information object
 * @param {string} imagesDir - Images directory path
 */
async function checkForImage(chapterInfo, imagesDir) {
  const imageExtensions = ['jpg', 'jpeg', 'png', 'gif'];
  
  // Try different image naming patterns
  const patterns = [
    `${chapterInfo.number}-${chapterInfo.name}`,  // e.g., 1.1-company
    `${chapterInfo.title.replace(/\s+/g, '')}`,   // e.g., Company
    `${chapterInfo.name}`                         // e.g., company
  ];
  
  for (const pattern of patterns) {
    for (const ext of imageExtensions) {
      const imageName = `${pattern}.${ext}`;
      const imagePath = path.join(imagesDir, imageName);
      
      try {
        await fs.access(imagePath);
        chapterInfo.fullPageImage = true;
        chapterInfo.imageName = imageName;
        return;
      } catch (err) {
        // Image doesn't exist, continue checking
      }
    }
  }
}

/**
 * Add file to hierarchical structure
 * @param {Object} structure - Book structure
 * @param {Object} fileInfo - File information
 */
function addToHierarchicalStructure(structure, fileInfo) {
  const numberParts = fileInfo.number.split('.');
  
  if (fileInfo.number.startsWith('0.')) {
    // Introduction files
    structure.introduction.files.push(fileInfo);
  } else if (fileInfo.number.startsWith('A.')) {
    // Appendix files
    structure.appendix.files.push(fileInfo);
  } else if (numberParts.length >= 2) {
    // Part files (1.x, 2.x, etc.)
    const partNum = parseInt(numberParts[0]);
    const part = structure.parts.find(p => p.number === partNum);
    
    if (part) {
      if (numberParts.length === 2) {
        // Chapter level (e.g., 2.1-attract.md = Part 2, Chapter 1)
        const chapterNum = parseInt(numberParts[1]);
        const chapter = {
          ...fileInfo,
          chapterNumber: chapterNum,
          partNumber: partNum,
          sections: [],
          isChapter: true
        };
        part.chapters.push(chapter);
      } else {
        // Section level (e.g., 2.1.1-attract-introduction.md = Part 2, Chapter 1, Section 1)
        const chapterNum = parseInt(numberParts[1]);
        const chapterKey = `${partNum}.${chapterNum}`;
        let parentChapter = part.chapters.find(c => c.chapterNumber === chapterNum);
        
        if (!parentChapter) {
          // Create placeholder chapter if it doesn't exist
          parentChapter = {
            number: chapterKey,
            chapterNumber: chapterNum,
            partNumber: partNum,
            name: 'placeholder',
            title: 'Placeholder Chapter',
            file: '',
            sections: [],
            isChapter: true,
            fullPageImage: false
          };
          part.chapters.push(parentChapter);
        }
        
        const sectionNum = parseInt(numberParts[2]);
        parentChapter.sections.push({
          ...fileInfo,
          sectionNumber: sectionNum,
          chapterNumber: chapterNum,
          partNumber: partNum,
          isSection: true
        });
      }
    }
  }
}

/**
 * Legacy function for backward compatibility
 * @param {string} contentDir - Path to content directory
 * @param {string} imagesDir - Path to images directory
 * @returns {Object} Detected book structure
 */
async function detectChapterStructure(contentDir, imagesDir) {
  // Try hierarchical structure first
  const hierarchicalStructure = await detectHierarchicalBookStructure(contentDir, imagesDir);
  
  // Convert to legacy format for backward compatibility
  const legacyStructure = {
    chapters: [],
    images: hierarchicalStructure.images,
    sections: {}
  };
  
  // Flatten hierarchical structure to legacy format
  hierarchicalStructure.parts.forEach(part => {
    part.chapters.forEach(chapter => {
      if (chapter.isChapter) {
        legacyStructure.chapters.push({
          file: chapter.file,
          number: parseFloat(chapter.number),
          name: chapter.name,
          title: chapter.title,
          titlePage: true,
          fullPageImage: chapter.fullPageImage,
          sections: chapter.sections.map(section => ({
            file: section.file,
            chapterNumber: parseInt(chapter.number.split('.')[0]),
            sectionNumber: parseInt(section.number.split('.').slice(-1)[0]),
            name: section.name,
            title: section.title,
            fullPath: section.number
          }))
        });
        
        // Add sections to legacy sections object
        chapter.sections.forEach(section => {
          legacyStructure.sections[section.file] = {
            file: section.file,
            chapterNumber: parseInt(chapter.number.split('.')[0]),
            sectionNumber: parseInt(section.number.split('.').slice(-1)[0]),
            name: section.name,
            title: section.title,
            fullPath: section.number
          };
        });
      }
    });
  });
  
  return legacyStructure;
}

/**
 * Process a hierarchical file (part, chapter, or section) with appropriate formatting
 * @param {Object} item - File item (part, chapter, or section)
 * @param {string} content - File content
 * @param {string} imagesDir - Images directory path
 * @param {string} type - Type of item ('part', 'chapter', 'section', 'introduction', 'appendix')
 * @returns {string} Processed content
 */
function processHierarchicalItem(item, content, imagesDir, type = 'chapter') {
  let processed = '';
  
  if (type === 'part') {
    // Add part title page
    processed += `\\newpart{${item.number}}{${item.title}}\n\n`;
  } else if (type === 'chapter') {
    // Add chapter title page marker with simple chapter number
    const chapterNum = item.chapterNumber || item.number;
    processed += `\\newchapter{${chapterNum}}{${item.title}}\n\n`;
  } else if (type === 'section') {
    // Add section header with simple section number
    const sectionNum = item.sectionNumber || item.number;
    processed += `\\newsection{${sectionNum}}{${item.title}}\n\n`;
  }
  
  // Add full-page image only if it exists
  if (item.fullPageImage && item.imageName) {
    const imagePath = path.join(imagesDir, item.imageName);
    // Verify the image file exists before including it
    try {
      accessSync(imagePath);
      processed += `![](${imagePath})\n\n`;
    } catch (err) {
      console.warn(`Image not found: ${imagePath} for ${type} ${item.number}`);
      // Don't include image if it doesn't exist
    }
  }
  
  // Process content for image references
  const processedContent = processImageReferences(content, item.file, imagesDir);
  processed += processedContent;
  
  return processed;
}

/**
 * Legacy function for backward compatibility
 * @param {Object} chapter - Chapter object
 * @param {string} content - Chapter content
 * @param {string} imagesDir - Images directory path
 * @returns {string} Processed content
 */
function processChapter(chapter, content, imagesDir) {
  return processHierarchicalItem(chapter, content, imagesDir, 'chapter');
}

/**
 * Process image references in markdown content
 * @param {string} content - Markdown content
 * @param {string} currentFile - Current file being processed
 * @param {string} imagesDir - Images directory path
 * @returns {string} Content with resolved image paths
 */
function processImageReferences(content, currentFile, imagesDir) {
  // First, handle Unicode characters that LaTeX doesn't handle well
  content = content.replace(/✓/g, '[X]');
  content = content.replace(/✗/g, '[ ]');
  content = content.replace(/→/g, '->');
  content = content.replace(/←/g, '<-');
  content = content.replace(/↑/g, '^');
  content = content.replace(/↓/g, 'v');
  content = content.replace(/•/g, '*');
  content = content.replace(/…/g, '...');
  
  // Then process image references
  return content.replace(
    /!\[([^\]]*)\]\(([^)]+)\)/g,
    (match, alt, src) => {
      // If already absolute path or URL, leave as is
      if (src.startsWith('/') || src.startsWith('http')) {
        return match;
      }
      
      // Handle images/ path references by converting to images/
      let imagePath = src;
      if (src.startsWith('images/')) {
        imagePath = src.replace('images/', '');
      }
      
      // Resolve relative path to images directory
      const fullPath = path.join(imagesDir, imagePath);
      
      // Check if file exists  
      try {
        // Use synchronous access for now since we're in a replace callback
        accessSync(fullPath);
        return `![${alt}](${fullPath})`;
      } catch (err) {
        console.warn(`Image not found: ${src} in ${currentFile}`);
        // Return empty string or skip image if not found to prevent broken images
        return '';
      }
    }
  );
}

/**
 * Generate consolidated markdown for hierarchical book structure
 * @param {Object} structure - Hierarchical book structure
 * @param {string} contentDir - Content directory path
 * @param {string} imagesDir - Images directory path
 * @returns {string} Consolidated markdown content
 */
async function generateHierarchicalMarkdown(structure, contentDir, imagesDir) {
  let consolidated = '';
  
  // Process Introduction (just content, no chapter/part structure)
  if (structure.introduction.files.length > 0) {
    for (const file of structure.introduction.files) {
      if (file.file) {
        const filePath = path.join(file.folderPath, file.file);
        let content = await fs.readFile(filePath, 'utf8');
        content = stripYamlFrontmatter(content);
        
        // Convert markdown headers to proper LaTeX sections
        const lines = content.split('\n');
        const processedLines = [];
        
        for (const line of lines) {
          if (line.startsWith('# ')) {
            processedLines.push(`\\section*{${line.substring(2)}}`);
          } else if (line.startsWith('## ')) {
            processedLines.push(`\\subsection*{${line.substring(3)}}`);
          } else if (line.startsWith('### ')) {
            processedLines.push(`\\subsubsection*{${line.substring(4)}}`);
          } else {
            processedLines.push(line);
          }
        }
        
        content = processedLines.join('\n');
        
        // Remove CSS references from frontmatter to avoid conflicts
        content = content.replace(/^css:\s*.*$/gm, '');
        
        // Add full-page image if exists
        if (file.fullPageImage && file.imageName) {
          const imagePath = path.join(imagesDir, file.imageName);
          try {
            accessSync(imagePath);
            consolidated += `![](${imagePath})\n\n`;
          } catch (err) {
            // Image doesn't exist, skip
          }
        }
        
        // Process image references in introduction content
        const processedContent = processImageReferences(content, file.file, imagesDir);
        consolidated += processedContent + '\n\n';
      }
    }
  }
  
  // Process Parts
  for (const part of structure.parts) {
    if (part.chapters.length > 0) {
      consolidated += `# ${part.title}\n\n`;
      
      for (const chapter of part.chapters) {
        if (chapter.file) {
          const chapterPath = path.join(contentDir, part.folder, chapter.file);
          let chapterContent = await fs.readFile(chapterPath, 'utf8');
          chapterContent = stripYamlFrontmatter(chapterContent);
          
          // Remove markdown headers since we're using LaTeX structure
          chapterContent = chapterContent.replace(/^#+\s+/gm, '');
          
          // Remove CSS references from frontmatter to avoid conflicts
          chapterContent = chapterContent.replace(/^css:\s*.*$/gm, '');
          
          const chapterNum = chapter.chapterNumber || 1;
          consolidated += `# ${chapter.title}\n\n`;
          
          // Note: Images are now handled by the content files themselves
          // Removed automatic image insertion to prevent duplicates
          
          // Process image references in chapter content
          const processedChapterContent = processImageReferences(chapterContent, chapter.file, imagesDir);
          consolidated += processedChapterContent + '\n\n';
        }
        
        // Add sections
        for (const section of chapter.sections) {
          if (section.file) {
            const sectionPath = path.join(contentDir, part.folder, section.file);
            let sectionContent = await fs.readFile(sectionPath, 'utf8');
            sectionContent = stripYamlFrontmatter(sectionContent);
            
            // Remove markdown headers since we're using LaTeX structure
            sectionContent = sectionContent.replace(/^#+\s+/gm, '');
            
            const sectionNum = section.sectionNumber || 1;
            consolidated += `\\section{${section.title}}\n\n`;
            
            // Process image references in section content
            const processedSectionContent = processImageReferences(sectionContent, section.file, imagesDir);
            consolidated += processedSectionContent + '\n\n';
          }
        }
      }
    }
  }
  
  // Process Appendix
  if (structure.appendix.files.length > 0) {
    consolidated += '# Appendix\n\n';
    
    for (const file of structure.appendix.files) {
      if (file.file) {
        const filePath = path.join(file.folderPath, file.file);
        let content = await fs.readFile(filePath, 'utf8');
        content = stripYamlFrontmatter(content);
        
        // Remove markdown headers since we're using LaTeX structure
        content = content.replace(/^#+\s+/gm, '');
        
        consolidated += `# ${file.title}\n\n`;
        
        // Process image references in appendix content
        const processedContent = processImageReferences(content, file.file, imagesDir);
        consolidated += processedContent + '\n\n';
      }
    }
  }
  
  return consolidated;
}

/**
 * Legacy function for backward compatibility
 * @param {Object} structure - Book structure
 * @param {string} contentDir - Content directory path
 * @param {string} imagesDir - Images directory path
 * @returns {string} Consolidated markdown content
 */
async function generateConsolidatedMarkdown(structure, contentDir, imagesDir) {
  // Check if this is a hierarchical structure
  if (structure.parts && structure.introduction && structure.appendix) {
    return generateHierarchicalMarkdown(structure, contentDir, imagesDir);
  }
  
  // Legacy processing for old structure
  let consolidated = '';
  
  for (const chapter of structure.chapters) {
    // Read chapter content
    const chapterPath = path.join(contentDir, chapter.file);
    let chapterContent = await fs.readFile(chapterPath, 'utf8');
    chapterContent = stripYamlFrontmatter(chapterContent);
    
    // Process chapter with title page and image
    const processedChapter = processChapter(chapter, chapterContent, imagesDir);
    consolidated += processedChapter + '\n\n';
    
    // Add sections
    for (const section of chapter.sections) {
      const sectionPath = path.join(contentDir, section.file);
      let sectionContent = await fs.readFile(sectionPath, 'utf8');
      sectionContent = stripYamlFrontmatter(sectionContent);
      
      // Process section content
      const processedSection = processImageReferences(sectionContent, section.file, imagesDir);
      consolidated += processedSection + '\n\n';
    }
  }
  
  return consolidated;
}

/**
 * Create hierarchical table of contents from book structure
 * @param {Object} structure - Hierarchical book structure
 * @returns {string} Table of contents markdown
 */
function generateHierarchicalTableOfContents(structure) {
  let toc = '# Table of Contents\n\n';
  
  // Introduction
  if (structure.introduction.files.length > 0) {
    toc += '## Introduction\n\n';
    for (const file of structure.introduction.files) {
      // Extract title from filename or use formatted title
      const match = file.file.match(/^(\d+)-(.+)\.md$/);
      const title = match ? formatTitle(match[2]) : file.title || 'Introduction';
      toc += `- [${title}](#introduction)\n`;
    }
    toc += '\n';
  }
  
  // Parts
  for (const part of structure.parts) {
    if (part.chapters.length > 0) {
      toc += `## Part ${part.number}: ${part.title}\n\n`;
      
      for (const chapter of part.chapters) {
        if (chapter.file) {
          const chapterNum = chapter.number || chapter.chapterNumber;
          toc += `### ${chapterNum}: ${chapter.title}\n\n`;
        }
        
        // Group sections by level
        const level2Sections = chapter.sections?.filter(s => s.level === 2) || [];
        const level3Sections = chapter.sections?.filter(s => s.level === 3) || [];
        
        for (const section of level2Sections) {
          const sectionNum = section.number || section.sectionNumber;
          toc += `- [${sectionNum} ${section.title}](#${section.number.replace(/\./g, '-')})\n`;
        }
        
        for (const section of level3Sections) {
          const sectionNum = section.number || section.sectionNumber;
          toc += `  - [${sectionNum} ${section.title}](#${section.number.replace(/\./g, '-')})\n`;
        }
        
        if (chapter.sections && chapter.sections.length > 0) {
          toc += '\n';
        }
      }
    }
  }
  
  // Appendix
  if (structure.appendix.files.length > 0) {
    toc += '## Appendix\n\n';
    for (const file of structure.appendix.files) {
      const match = file.file.match(/^A\.(\d+)-(.+)\.md$/);
      const title = match ? formatTitle(match[2]) : file.title || 'Appendix';
      toc += `- [${title}](#appendix)\n`;
    }
    toc += '\n';
  }
  
  return toc;
}

/**
 * Legacy function for backward compatibility
 * @param {Object} structure - Book structure
 * @returns {string} Table of contents markdown
 */
function generateTableOfContents(structure) {
  // Check if this is a hierarchical structure
  if (structure.parts && structure.introduction && structure.appendix) {
    return generateHierarchicalTableOfContents(structure);
  }
  
  // Legacy TOC generation
  let toc = '# Table of Contents\n\n';
  
  for (const chapter of structure.chapters) {
    toc += `## Chapter ${chapter.number}: ${chapter.title}\n\n`;
    
    for (const section of chapter.sections) {
      toc += `### ${section.fullPath} ${section.title}\n\n`;
    }
  }
  
  return toc;
}

export {
  detectHierarchicalBookStructure,
  detectChapterStructure,
  processHierarchicalItem,
  processChapter,
  processImageReferences,
  generateHierarchicalMarkdown,
  generateConsolidatedMarkdown,
  generateHierarchicalTableOfContents,
  generateTableOfContents
};