UNPKG

codecrucible-synth

Version:

Production-Ready AI Development Platform with Multi-Voice Synthesis, Smithery MCP Integration, Enterprise Security, and Zero-Timeout Reliability

1,013 lines (834 loc) โ€ข 28.8 kB
#!/usr/bin/env node /** * Documentation Synchronization System * Automatically syncs and validates documentation across the codebase */ import { promises as fs, readFileSync } from 'fs'; import { glob } from 'glob'; import { resolve, relative, basename, dirname } from 'path'; import { logger } from './logger.js'; interface DocumentationFile { path: string; type: 'readme' | 'api' | 'guide' | 'changelog' | 'config' | 'other'; lastModified: number; size: number; content: string; references: string[]; outdated: boolean; missing: string[]; } interface DocumentationIssue { type: 'outdated' | 'missing' | 'broken_link' | 'inconsistent' | 'duplicate'; severity: 'low' | 'medium' | 'high' | 'critical'; file: string; description: string; suggestion: string; autoFixable: boolean; } interface DocumentationMetrics { totalFiles: number; outdatedFiles: number; missingDocumentation: number; brokenLinks: number; coverageScore: number; lastUpdated: number; } export class DocumentationSyncSystem { private docs: Map<string, DocumentationFile> = new Map(); private issues: DocumentationIssue[] = []; private codeFiles: string[] = []; /** * Analyze and synchronize all documentation */ async syncDocumentation(): Promise<{ metrics: DocumentationMetrics; issues: DocumentationIssue[]; recommendations: string[]; }> { logger.info('๐Ÿ“š Starting comprehensive documentation sync...'); // Discover all files await this.discoverFiles(); // Analyze documentation files await this.analyzeDocumentation(); // Check for missing documentation this.checkMissingDocumentation(); // Validate links and references await this.validateReferences(); // Check for outdated content this.checkOutdatedContent(); // Generate updated documentation await this.generateMissingDocs(); const metrics = this.calculateMetrics(); const recommendations = this.generateRecommendations(); return { metrics, issues: this.issues, recommendations, }; } /** * Discover all relevant files */ private async discoverFiles(): Promise<void> { // Documentation files const docPatterns = [ '*.md', 'docs/**/*.md', 'Docs/**/*.md', 'README*', 'CHANGELOG*', 'LICENSE*', '*.txt', ]; const docFiles: string[] = []; for (const pattern of docPatterns) { const files = await glob(pattern, { ignore: ['node_modules/**'] }); docFiles.push(...files); } // Code files for cross-referencing this.codeFiles = await glob('src/**/*.{ts,js}', { ignore: ['src/**/*.test.ts'] }); // Process documentation files for (const file of docFiles) { try { const stats = await fs.stat(file); const content = await fs.readFile(file, 'utf-8'); const doc: DocumentationFile = { path: file, type: this.classifyDocType(file), lastModified: stats.mtime.getTime(), size: stats.size, content, references: this.extractReferences(content), outdated: false, missing: [], }; this.docs.set(file, doc); } catch (error) { logger.warn(`Failed to process ${file}:`, error); } } logger.info( `๐Ÿ“„ Found ${this.docs.size} documentation files, ${this.codeFiles.length} code files` ); } /** * Classify documentation type */ private classifyDocType(filePath: string): DocumentationFile['type'] { const fileName = basename(filePath).toLowerCase(); if (fileName.includes('readme')) return 'readme'; if (fileName.includes('changelog') || fileName.includes('history')) return 'changelog'; if (fileName.includes('api') || fileName.includes('reference')) return 'api'; if (fileName.includes('guide') || fileName.includes('tutorial')) return 'guide'; if (fileName.includes('config') || fileName.includes('setup')) return 'config'; return 'other'; } /** * Extract references (links, code references, etc.) */ private extractReferences(content: string): string[] { const references: string[] = []; // Markdown links const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g; let match; while ((match = linkRegex.exec(content)) !== null) { references.push(match[2]); } // File references const fileRefRegex = /`([^`]+\.(ts|js|json|yaml|yml))`/g; while ((match = fileRefRegex.exec(content)) !== null) { references.push(match[1]); } // Code block references const codeBlockRegex = /```(\w+)\n([\s\S]*?)```/g; while ((match = codeBlockRegex.exec(content)) !== null) { // Extract import statements and function names const codeContent = match[2]; const importMatches = codeContent.match(/import.*from\s+['"]([^'"]+)['"]/g); if (importMatches) { references.push( ...importMatches.map(imp => imp.split('from')[1].trim().replace(/['"]/g, '')) ); } } return [...new Set(references)]; // Remove duplicates } /** * Analyze documentation for issues */ private async analyzeDocumentation(): Promise<void> { for (const [filePath, doc] of this.docs) { // Check for empty or minimal content if (doc.content.trim().length < 100) { this.issues.push({ type: 'missing', severity: 'medium', file: filePath, description: 'Documentation file is too short or empty', suggestion: 'Add comprehensive content explaining purpose and usage', autoFixable: false, }); } // Check for placeholder content const placeholders = ['TODO', 'TBD', 'Coming soon', 'Under construction']; for (const placeholder of placeholders) { if (doc.content.includes(placeholder)) { this.issues.push({ type: 'missing', severity: 'medium', file: filePath, description: `Contains placeholder: ${placeholder}`, suggestion: 'Replace placeholder with actual content', autoFixable: false, }); } } // Check for outdated timestamps const dateRegex = /\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{4}/g; const dates = doc.content.match(dateRegex); if (dates) { const oldestDate = new Date(Math.min(...dates.map(d => new Date(d).getTime()))); const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); if (oldestDate < oneYearAgo) { doc.outdated = true; this.issues.push({ type: 'outdated', severity: 'low', file: filePath, description: 'Contains dates older than one year', suggestion: 'Review and update dated information', autoFixable: false, }); } } } } /** * Check for missing documentation */ private checkMissingDocumentation(): void { // Check for main documentation files const requiredDocs = [ { file: 'README.md', description: 'Main project README' }, { file: 'SETUP.md', description: 'Setup and installation guide' }, { file: 'CHANGELOG.md', description: 'Change history' }, { file: 'docs/API.md', description: 'API documentation' }, ]; for (const required of requiredDocs) { if (!this.docs.has(required.file) && !this.docs.has(required.file.toLowerCase())) { this.issues.push({ type: 'missing', severity: 'high', file: required.file, description: `Missing ${required.description}`, suggestion: `Create ${required.file} with comprehensive ${required.description.toLowerCase()}`, autoFixable: true, }); } } // Check for code files without documentation const criticalFiles = this.codeFiles.filter( file => file.includes('index.ts') || file.includes('main.ts') || file.includes('cli.ts') || file.includes('client.ts') || file.includes('agent.ts') ); for (const file of criticalFiles) { const hasInlineDoc = this.hasInlineDocumentation(file); const hasExternalDoc = this.hasExternalDocumentation(file); if (!hasInlineDoc && !hasExternalDoc) { this.issues.push({ type: 'missing', severity: 'medium', file: file, description: 'Critical file lacks documentation', suggestion: 'Add JSDoc comments or create corresponding documentation file', autoFixable: true, }); } } } /** * Check if file has inline documentation */ private hasInlineDocumentation(filePath: string): boolean { try { const content = readFileSync(filePath, 'utf-8'); return content.includes('/**') || (content.includes('//') && content.split('//').length > 5); } catch { return false; } } /** * Check if file has external documentation */ private hasExternalDocumentation(filePath: string): boolean { const fileName = basename(filePath, '.ts').replace('.js', ''); for (const doc of this.docs.values()) { if (doc.content.toLowerCase().includes(fileName.toLowerCase())) { return true; } } return false; } /** * Validate references and links */ private async validateReferences(): Promise<void> { for (const [filePath, doc] of this.docs) { for (const ref of doc.references) { // Skip external links if (ref.startsWith('http://') || ref.startsWith('https://')) { continue; } // Check file references if (ref.includes('.ts') || ref.includes('.js') || ref.includes('.json')) { const referencedPath = resolve(dirname(filePath), ref); try { await fs.access(referencedPath); } catch { this.issues.push({ type: 'broken_link', severity: 'medium', file: filePath, description: `Broken file reference: ${ref}`, suggestion: 'Update file path or remove broken reference', autoFixable: false, }); } } // Check markdown file references if (ref.endsWith('.md')) { const referencedPath = resolve(dirname(filePath), ref); if (!this.docs.has(referencedPath) && !this.docs.has(ref)) { this.issues.push({ type: 'broken_link', severity: 'medium', file: filePath, description: `Broken documentation link: ${ref}`, suggestion: 'Create referenced documentation or fix link', autoFixable: true, }); } } } } } /** * Check for outdated content */ private checkOutdatedContent(): void { const packageJsonPath = 'package.json'; let packageVersion = '1.0.0'; try { const packageContent = readFileSync(packageJsonPath, 'utf-8'); const packageData = JSON.parse(packageContent); packageVersion = packageData.version; } catch { // Ignore if package.json not found } for (const [filePath, doc] of this.docs) { // Check for version mismatches const versionRegex = /version\s*:?\s*['"]?([0-9]+\.[0-9]+\.[0-9]+)/gi; const versionMatches = doc.content.match(versionRegex); if (versionMatches) { for (const versionMatch of versionMatches) { const extractedVersion = versionMatch.match(/([0-9]+\.[0-9]+\.[0-9]+)/)?.[1]; if (extractedVersion && extractedVersion !== packageVersion) { this.issues.push({ type: 'outdated', severity: 'medium', file: filePath, description: `Version mismatch: shows ${extractedVersion}, current is ${packageVersion}`, suggestion: `Update version references to ${packageVersion}`, autoFixable: true, }); } } } // Check for outdated code examples this.checkCodeExamples(filePath, doc); } } /** * Check code examples in documentation */ private checkCodeExamples(filePath: string, doc: DocumentationFile): void { const codeBlockRegex = /```(?:typescript|javascript|ts|js)\n([\s\S]*?)```/g; let match; while ((match = codeBlockRegex.exec(doc.content)) !== null) { const codeContent = match[1]; // Check for imports that don't exist const importRegex = /import.*from\s+['"]([^'"]+)['"]/g; let importMatch; while ((importMatch = importRegex.exec(codeContent)) !== null) { const importPath = importMatch[1]; // Check if import path exists in codebase const pathExists = this.codeFiles.some( file => file.includes(importPath) || file.replace('src/', '').replace('.ts', '').includes(importPath) ); if (!pathExists && !importPath.startsWith('.') && !importPath.startsWith('node:')) { this.issues.push({ type: 'outdated', severity: 'medium', file: filePath, description: `Code example references non-existent import: ${importPath}`, suggestion: 'Update code example with correct import paths', autoFixable: false, }); } } } } /** * Generate missing documentation */ private async generateMissingDocs(): Promise<void> { const autoFixableIssues = this.issues.filter(i => i.autoFixable); for (const issue of autoFixableIssues) { try { if (issue.type === 'missing' && issue.file.endsWith('.md')) { await this.generateDocFile(issue.file); } } catch (error) { logger.warn(`Failed to generate ${issue.file}:`, error); } } } /** * Generate a documentation file */ private async generateDocFile(filePath: string): Promise<void> { let content = ''; if (filePath.includes('README')) { content = await this.generateReadme(); } else if (filePath.includes('API')) { content = await this.generateApiDocs(); } else if (filePath.includes('SETUP')) { content = await this.generateSetupGuide(); } else if (filePath.includes('CHANGELOG')) { content = await this.generateChangelog(); } if (content) { // Ensure directory exists await fs.mkdir(dirname(filePath), { recursive: true }); await fs.writeFile(filePath, content); logger.info(`๐Ÿ“ Generated documentation: ${filePath}`); } } /** * Generate README content */ private async generateReadme(): Promise<string> { let packageData: Record<string, unknown> = {}; try { const packageContent = await fs.readFile('package.json', 'utf-8'); packageData = JSON.parse(packageContent); } catch { // Use defaults if package.json not found } return `# ${packageData.name || 'Project Name'} ${packageData.description || 'A comprehensive AI-powered development tool.'} ## Features - ๐Ÿค– AI-powered code generation and analysis - ๐Ÿ”’ Enterprise-grade security and sandboxing - โšก High-performance local and cloud model support - ๐Ÿงช Comprehensive testing and CI/CD integration - ๐Ÿ“Š Advanced observability and error handling - ๐Ÿ”ง Extensible architecture with plugin support ## Quick Start \`\`\`bash npm install ${packageData.name || 'project-name'} \`\`\` \`\`\`bash # Basic usage npx ${packageData.name || 'project-name'} --help \`\`\` ## Documentation - [Setup Guide](SETUP.md) - Installation and configuration - [API Documentation](docs/API.md) - Complete API reference - [Configuration Guide](docs/Configuration.md) - Configuration options - [Examples](docs/examples/) - Usage examples and tutorials ## Requirements - Node.js ${(packageData.engines as any)?.node || '>=18.0.0'} - TypeScript ${(packageData.devDependencies as any)?.typescript || '>=5.0.0'} ## License ${packageData.license || 'MIT'} ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines. --- Generated automatically by Documentation Sync System Last updated: ${new Date().toISOString()} `; } /** * Generate API documentation */ private async generateApiDocs(): Promise<string> { const exports = await this.extractExports(); let content = `# API Documentation ## Overview This document provides a comprehensive reference for all public APIs. ## Modules `; for (const [moduleName, moduleExports] of exports) { content += `### ${moduleName}\n\n`; for (const exportItem of moduleExports) { content += `#### \`${exportItem.name}\`\n\n`; content += `${exportItem.description || 'No description available.'}\n\n`; if (exportItem.type === 'class') { content += `**Type:** Class\n\n`; } else if (exportItem.type === 'function') { content += `**Type:** Function\n\n`; } content += `**Usage:**\n\`\`\`typescript\n// TODO: Add usage example\n\`\`\`\n\n`; } } content += `\n---\n\nGenerated automatically by Documentation Sync System\nLast updated: ${new Date().toISOString()}\n`; return content; } /** * Extract exports from code files */ private async extractExports(): Promise< Map<string, Array<{ name: string; type: string; description?: string }>> > { const exports = new Map(); for (const file of this.codeFiles) { try { const content = await fs.readFile(file, 'utf-8'); const moduleName = relative('src', file).replace('.ts', '').replace('.js', ''); const moduleExports = []; // Extract class exports const classRegex = /export\s+class\s+(\w+)/g; let match; while ((match = classRegex.exec(content)) !== null) { moduleExports.push({ name: match[1], type: 'class' }); } // Extract function exports const functionRegex = /export\s+(?:async\s+)?function\s+(\w+)/g; while ((match = functionRegex.exec(content)) !== null) { moduleExports.push({ name: match[1], type: 'function' }); } if (moduleExports.length > 0) { exports.set(moduleName, moduleExports); } } catch (error) { // Ignore files that can't be read } } return exports; } /** * Generate setup guide */ private async generateSetupGuide(): Promise<string> { return `# Setup Guide ## Installation ### Prerequisites - Node.js 18.0.0 or higher - npm or yarn package manager ### Install from npm \`\`\`bash npm install -g codecrucible-synth \`\`\` ### Install from source \`\`\`bash git clone https://github.com/rhinos0608/codecrucible-synth.git cd codecrucible-synth npm install npm run build npm link \`\`\` ## Configuration ### Environment Variables Create a \`.env\` file in your project root: \`\`\`bash # Model Configuration OLLAMA_ENDPOINT=http://localhost:11434 LMSTUDIO_ENDPOINT=http://localhost:1234 # Security Settings ENABLE_SANDBOX=true MAX_INPUT_LENGTH=50000 # Performance Settings CACHE_ENABLED=true MAX_CONCURRENT_REQUESTS=3 \`\`\` ### Configuration Files The system supports multiple configuration formats: - \`codecrucible.config.json\` - Main configuration - \`config/default.yaml\` - Default settings - \`config/hybrid.yaml\` - Hybrid model settings ## First Run \`\`\`bash # Test installation codecrucible --help # Initialize project codecrucible init # Run with default settings codecrucible "Create a hello world function" \`\`\` ## Troubleshooting ### Common Issues 1. **Permission Errors**: Run with appropriate permissions 2. **Model Not Found**: Ensure local models are downloaded 3. **Network Issues**: Check firewall and proxy settings ### Debug Mode \`\`\`bash codecrucible --debug --verbose "your command" \`\`\` --- Generated automatically by Documentation Sync System Last updated: ${new Date().toISOString()} `; } /** * Generate changelog */ private async generateChangelog(): Promise<string> { return `# Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added - Comprehensive error handling and observability system - Memory leak detection and performance optimization - Automated documentation synchronization - Security hardening and audit tools - Multi-user scalability preparation ### Changed - Improved TypeScript type safety across codebase - Consolidated configuration system - Enhanced test coverage and CI/CD pipeline ### Fixed - Memory leaks in workflow orchestrator - Security vulnerabilities in sandbox execution - Configuration conflicts across multiple files ## [3.5.0] - ${new Date().toISOString().split('T')[0]} ### Added - Initial release with core functionality - AI-powered code generation - Local and cloud model support - Security sandbox execution --- Generated automatically by Documentation Sync System Last updated: ${new Date().toISOString()} `; } /** * Calculate documentation metrics */ private calculateMetrics(): DocumentationMetrics { const totalFiles = this.docs.size; const outdatedFiles = Array.from(this.docs.values()).filter(d => d.outdated).length; const missingDocumentation = this.issues.filter(i => i.type === 'missing').length; const brokenLinks = this.issues.filter(i => i.type === 'broken_link').length; // Coverage score based on critical files having documentation const criticalFiles = this.codeFiles.filter( f => f.includes('index.ts') || f.includes('main.ts') || f.includes('cli.ts') ); const documentedCriticalFiles = criticalFiles.filter( f => this.hasInlineDocumentation(f) || this.hasExternalDocumentation(f) ); const coverageScore = criticalFiles.length > 0 ? Math.round((documentedCriticalFiles.length / criticalFiles.length) * 100) : 100; const lastUpdated = Math.max(...Array.from(this.docs.values()).map(d => d.lastModified)); return { totalFiles, outdatedFiles, missingDocumentation, brokenLinks, coverageScore, lastUpdated, }; } /** * Generate recommendations */ private generateRecommendations(): string[] { const recommendations: string[] = []; const metrics = this.calculateMetrics(); if (metrics.coverageScore < 80) { recommendations.push('๐Ÿ“ Improve documentation coverage by adding docs to critical files'); } if (metrics.outdatedFiles > 0) { recommendations.push(`๐Ÿ“… Update ${metrics.outdatedFiles} outdated documentation files`); } if (metrics.brokenLinks > 0) { recommendations.push(`๐Ÿ”— Fix ${metrics.brokenLinks} broken links and references`); } if (metrics.missingDocumentation > 0) { recommendations.push(`๐Ÿ“„ Create ${metrics.missingDocumentation} missing documentation files`); } recommendations.push('๐Ÿ”„ Set up automated documentation generation in CI/CD'); recommendations.push('๐Ÿ“Š Add documentation metrics to project dashboard'); recommendations.push('๐ŸŽฏ Establish documentation review process for code changes'); return recommendations; } /** * Generate comprehensive report */ generateReport(): string { const metrics = this.calculateMetrics(); let report = ` ๐Ÿ“š DOCUMENTATION SYNCHRONIZATION REPORT ====================================== OVERVIEW: โ€ข Total Documentation Files: ${metrics.totalFiles} โ€ข Coverage Score: ${metrics.coverageScore}% โ€ข Outdated Files: ${metrics.outdatedFiles} โ€ข Missing Documentation: ${metrics.missingDocumentation} โ€ข Broken Links: ${metrics.brokenLinks} `; // Issues by type const issueTypes = new Map<string, DocumentationIssue[]>(); for (const issue of this.issues) { if (!issueTypes.has(issue.type)) { issueTypes.set(issue.type, []); } issueTypes.get(issue.type)!.push(issue); } if (this.issues.length > 0) { report += 'ISSUES FOUND:\n'; report += `${'โ”€'.repeat(50)}\n`; for (const [type, issues] of issueTypes) { const icon = type === 'missing' ? '๐Ÿ“„' : type === 'outdated' ? '๐Ÿ“…' : type === 'broken_link' ? '๐Ÿ”—' : type === 'inconsistent' ? 'โš ๏ธ' : '๐Ÿ”„'; report += `${icon} ${type.replace('_', ' ').toUpperCase()} (${issues.length}):\n`; for (const issue of issues.slice(0, 5)) { const severity = issue.severity === 'critical' ? '๐Ÿšจ' : issue.severity === 'high' ? '๐Ÿ”ด' : issue.severity === 'medium' ? '๐ŸŸก' : '๐Ÿ”ต'; report += ` ${severity} ${issue.file}: ${issue.description}\n`; report += ` ๐Ÿ’ก ${issue.suggestion}\n`; } if (issues.length > 5) { report += ` ... and ${issues.length - 5} more\n`; } report += '\n'; } } // Documentation files by type const docsByType = new Map<string, number>(); for (const doc of this.docs.values()) { docsByType.set(doc.type, (docsByType.get(doc.type) || 0) + 1); } report += 'DOCUMENTATION BREAKDOWN:\n'; report += `${'โ”€'.repeat(50)}\n`; for (const [type, count] of docsByType) { const icon = type === 'readme' ? '๐Ÿ“–' : type === 'api' ? '๐Ÿ”ง' : type === 'guide' ? '๐Ÿ“š' : type === 'changelog' ? '๐Ÿ“…' : '๐Ÿ“„'; report += `${icon} ${type}: ${count}\n`; } return report; } /** * Apply automatic fixes */ async applyAutomaticFixes(): Promise<{ fixed: number; failed: number }> { let fixed = 0; let failed = 0; const autoFixableIssues = this.issues.filter(i => i.autoFixable); for (const issue of autoFixableIssues) { try { if (issue.type === 'missing' && issue.file.endsWith('.md')) { await this.generateDocFile(issue.file); fixed++; } else if (issue.type === 'outdated' && issue.description.includes('version')) { await this.fixVersionReferences(issue.file); fixed++; } } catch (error) { failed++; logger.error(`Failed to fix ${issue.file}:`, error); } } return { fixed, failed }; } /** * Fix version references in documentation */ private async fixVersionReferences(filePath: string): Promise<void> { try { const packageContent = await fs.readFile('package.json', 'utf-8'); const packageData = JSON.parse(packageContent); const currentVersion = packageData.version; let content = await fs.readFile(filePath, 'utf-8'); // Replace version references const versionRegex = /version\s*:?\s*['"]?[0-9]+\.[0-9]+\.[0-9]+['"]?/gi; content = content.replace(versionRegex, `version: "${currentVersion}"`); await fs.writeFile(filePath, content); logger.info(`โœ… Fixed version references in ${filePath}`); } catch (error) { throw new Error(`Failed to fix version references: ${error}`); } } /** * Dispose of the documentation sync system */ dispose(): void { this.docs.clear(); this.issues = []; this.codeFiles = []; logger.info('๐Ÿ—‘๏ธ Documentation sync system disposed'); } } // CLI usage if (import.meta.url === `file://${process.argv[1]}`) { const docSync = new DocumentationSyncSystem(); docSync .syncDocumentation() .then(result => { console.log(docSync.generateReport()); if (result.issues.length > 0) { console.log(`\n๐Ÿ”ง Found ${result.issues.length} documentation issues`); console.log('๐Ÿ“‹ Recommendations:'); for (const rec of result.recommendations) { console.log(` โ€ข ${rec}`); } const autoFixable = result.issues.filter(i => i.autoFixable).length; if (autoFixable > 0) { console.log(`\n๐Ÿ’พ ${autoFixable} issues can be automatically fixed`); } } else { console.log('\nโœ… Documentation is up to date!'); } }) .finally(() => { docSync.dispose(); }) .catch(error => { console.error('Documentation sync failed:', error); docSync.dispose(); process.exit(1); }); }