@afterxleep/doc-bot/src/index.js

Generic MCP server for intelligent documentation access in any project

import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { CallToolRequestSchema, ListResourcesRequestSchema, ListToolsRequestSchema, ReadResourceRequestSchema } from '@modelcontextprotocol/sdk/types.js';
import { DocumentationService } from './services/DocumentationService.js';
import { InferenceEngine } from './services/InferenceEngine.js';
import { MultiDocsetDatabase } from './services/docset/database.js';
import { DocsetService } from './services/docset/index.js';
import { UnifiedSearchService } from './services/UnifiedSearchService.js';
import chokidar from 'chokidar';
import path from 'path';
import { promises as fs } from 'fs';
import { fileURLToPath } from 'url';
import os from 'os';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

class DocsServer {
  constructor(options = {}) {
    this.options = {
      docsPath: options.docsPath || './doc-bot',
      verbose: options.verbose || false,
      watch: options.watch || false,
      ...options
    };
    
    // Cache for prompt templates
    this.promptTemplates = {};
    
    // Track search attempts per session/query pattern
    this.searchAttempts = new Map();
    
    this.server = new Server({
      name: 'doc-bot',
      version: '1.0.0',
      description: 'Generic MCP server for intelligent documentation access'
    }, {
      capabilities: {
        resources: {},
        tools: {}
      }
    });
    
    this.docService = new DocumentationService(this.options.docsPath);
    this.inferenceEngine = new InferenceEngine(this.docService);
    
    // Initialize docset services
    const docsetsPath = this.options.docsetsPath || path.join(os.homedir(), 'Developer', 'DocSets');
    this.docsetService = new DocsetService(docsetsPath);
    this.multiDocsetDb = new MultiDocsetDatabase();
    
    // Initialize unified search
    this.unifiedSearch = new UnifiedSearchService(this.docService, this.multiDocsetDb);
    
    this.setupHandlers();
    
    if (this.options.watch) {
      this.setupWatcher();
    }
  }

  async loadPromptTemplate(templateName) {
    if (!this.promptTemplates[templateName]) {
      const templatePath = path.join(__dirname, '../prompts', `${templateName}.txt`);
      try {
        this.promptTemplates[templateName] = await fs.readFile(templatePath, 'utf8');
      } catch (error) {
        console.error(`Failed to load prompt template ${templateName}:`, error);
        return null;
      }
    }
    return this.promptTemplates[templateName];
  }
  
  setupHandlers() {
    // List available resources
    this.server.setRequestHandler(ListResourcesRequestSchema, async () => {
      return {
        resources: [
          {
            uri: 'docs://search',
            name: 'Search Documentation',
            description: 'Powerful search across all your project documentation with intelligent ranking',
            mimeType: 'application/json'
          },
          {
            uri: 'docs://global-rules',
            name: 'Global Rules',
            description: 'Access your project\'s core standards and best practices',
            mimeType: 'application/json'
          },
          {
            uri: 'docs://contextual',
            name: 'Contextual Documentation',
            description: 'Smart documentation suggestions based on your current context',
            mimeType: 'application/json'
          },
          {
            uri: 'docs://system-prompt',
            name: 'System Prompt Injection',
            description: 'Enhanced AI capabilities powered by your project\'s knowledge base',
            mimeType: 'text/plain'
          }
        ]
      };
    });
    
    // Read resource content
    this.server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
      const { uri } = request.params;
      
      switch (uri) {
        case 'docs://search':
          const allDocs = await this.docService.getAllDocuments();
          return {
            contents: [{
              uri,
              mimeType: 'application/json',
              text: JSON.stringify(allDocs, null, 2)
            }]
          };
          
        case 'docs://global-rules':
          const globalRules = await this.docService.getGlobalRules();
          return {
            contents: [{
              uri,
              mimeType: 'application/json',
              text: JSON.stringify(globalRules, null, 2)
            }]
          };
          
        case 'docs://system-prompt':
          const systemPrompt = await this.generateSystemPrompt();
          return {
            contents: [{
              uri,
              mimeType: 'text/plain',
              text: systemPrompt
            }]
          };
          
        default:
          throw new Error(`Unknown resource: ${uri}`);
      }
    });
    
    // List available tools
    this.server.setRequestHandler(ListToolsRequestSchema, async () => {
      return {
        tools: [
          {
            name: 'check_project_rules',
            description: 'Get mandatory coding standards for your task. Always call before writing code. Returns architecture patterns, security requirements, and performance guidelines specific to this codebase. Critical for maintaining code quality and consistency.',
            inputSchema: {
              type: 'object',
              properties: {
                task: {
                  type: 'string',
                  description: 'Your coding task in 2-5 words. Examples: "REST API endpoint", "authentication service", "React component", "database migration"'
                }
              },
              required: ['task']
            }
          },
          {
            name: 'search_documentation',
            description: 'Search codebase documentation and API references. Searches project patterns, architecture decisions, and API docs. Best results with technical terms like class names, not descriptions. Examples: search "Widget" not "iOS 18 features".',
            inputSchema: {
              type: 'object',
              properties: {
                query: {
                  type: 'string',
                  description: 'Technical search terms. Use API/class names, not descriptions. Good: "URLSession", "WidgetKit", "CoreData". Bad: "how to make network calls"'
                },
                limit: {
                  type: 'number',
                  description: 'Maximum results to return. Default: 20'
                },
                docsetId: {
                  type: 'string',
                  description: 'Filter results to specific documentation set'
                },
                type: {
                  type: 'string',
                  description: 'Filter API results by type: "Class", "Method", "Function", "Property", "Framework", "Protocol", "Enum"'
                }
              },
              required: ['query']
            }
          },
          {
            name: 'get_global_rules',
            description: 'Get comprehensive coding standards and architecture guidelines. Returns project-wide engineering principles, design patterns, performance requirements, and security standards. Essential reading for understanding the codebase philosophy.',
            inputSchema: {
              type: 'object',
              properties: {},
              additionalProperties: false
            }
          },
          {
            name: 'get_file_docs',
            description: 'Get file-specific coding patterns and conventions. Returns contextual guidelines, performance considerations, and architectural decisions for specific files or directories. Use before modifying existing code.',
            inputSchema: {
              type: 'object',
              properties: {
                filePath: {
                  type: 'string',
                  description: 'File path or pattern. Examples: "src/components/Button.tsx", "**/*.test.js", "services/auth/*"'
                }
              },
              required: ['filePath']
            }
          },
          {
            name: 'read_specific_document',
            description: 'Read full documentation file content. Use after search_documentation to get complete implementation details, code examples, and architectural decisions. Only for project docs, not API docs.',
            inputSchema: {
              type: 'object',
              properties: {
                fileName: {
                  type: 'string',
                  description: 'Name of the documentation file to read. Must match exactly. Example: "coding-standards.md"'
                }
              },
              required: ['fileName']
            }
          },
          {
            name: 'explore_api',
            description: 'Deep dive into any API, framework, or class. Returns all methods, properties, protocols, and usage examples. Essential for understanding how to implement features correctly. Much faster than multiple searches.',
            inputSchema: {
              type: 'object',
              properties: {
                apiName: {
                  type: 'string',
                  description: 'API, framework, or class name. Examples: "URLSession", "WidgetKit", "SwiftUI.View", "React.Component"'
                },
                docsetId: {
                  type: 'string',
                  description: 'Limit exploration to specific documentation set'
                }
              },
              required: ['apiName']
            }
          },
          {
            name: 'create_or_update_rule',
            description: 'Document new coding patterns or architectural decisions. Captures lessons learned, design patterns, and team conventions as searchable knowledge. Great for preserving tribal knowledge.',
            inputSchema: {
              type: 'object',
              properties: {
                fileName: {
                  type: 'string',
                  description: 'Documentation file name. Must end with .md. Example: "api-patterns.md"'
                },
                title: {
                  type: 'string',
                  description: 'Document title for display and search'
                },
                description: {
                  type: 'string',
                  description: 'Brief summary of the document\'s purpose'
                },
                keywords: {
                  type: 'array',
                  items: { type: 'string' },
                  description: 'Search keywords. Include technologies, patterns, and concepts covered'
                },
                alwaysApply: {
                  type: 'boolean',
                  description: 'true: applies to all code (global rule). false: applies only when relevant (contextual)'
                },
                content: {
                  type: 'string',
                  description: 'Full markdown content of the documentation'
                }
              },
              required: ['fileName', 'title', 'content', 'alwaysApply']
            }
          },
          {
            name: 'refresh_documentation',
            description: 'Reload all project documentation from disk. Use when documentation files have been modified outside of this session. Re-indexes all documents and updates search index.',
            inputSchema: {
              type: 'object',
              properties: {},
              additionalProperties: false
            }
          },
          {
            name: 'get_document_index',
            description: 'List all available project documentation files. Returns index with titles, descriptions, and metadata. Use to see what documentation exists without searching.',
            inputSchema: {
              type: 'object',
              properties: {},
              additionalProperties: false
            }
          },
          {
            name: 'add_docset',
            description: 'Install a new documentation set (docset) for API reference. Supports both local .docset files and direct URLs. Docsets provide official API documentation for frameworks and libraries.',
            inputSchema: {
              type: 'object',
              properties: {
                source: {
                  type: 'string',
                  description: 'Path to local .docset file/directory or URL to download. Examples: "/Downloads/Swift.docset", "https://example.com/React.docset.tgz"'
                }
              },
              required: ['source']
            }
          },
          {
            name: 'remove_docset',
            description: 'Remove an installed documentation set. Use list_docsets first to see available docsets and their IDs.',
            inputSchema: {
              type: 'object',
              properties: {
                docsetId: {
                  type: 'string',
                  description: 'ID of the docset to remove. Get this from list_docsets command.'
                }
              },
              required: ['docsetId']
            }
          },
          {
            name: 'list_docsets',
            description: 'List all installed documentation sets (docsets). Shows docset IDs, names, and installation details. Use this to see what API documentation is available.',
            inputSchema: {
              type: 'object',
              properties: {},
              additionalProperties: false
            }
          },
          {
            name: 'doc_bot',
            description: 'Your intelligent project assistant. Analyzes ANY request and provides smart routing to the right tools. ALWAYS call this first - it understands coding, documentation, architecture questions, and more.',
            inputSchema: {
              type: 'object',
              properties: {
                task: {
                  type: 'string',
                  description: 'What do you need help with? Examples: "create REST API", "understand auth flow", "document this pattern", "find database models"'
                }
              },
              required: ['task']
            }
          }
        ]
      };
    });
    
    // Handle tool calls
    this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
      const { name, arguments: args } = request.params;
      
      try {
        switch (name) {
          case 'check_project_rules':
            const task = args?.task || 'code generation';
            const mandatoryRules = await this.getMandatoryRules(task);
            return {
              content: [{
                type: 'text',
                text: mandatoryRules
              }]
            };
            
          case 'search_documentation':
            const unifiedQuery = args?.query;
            if (!unifiedQuery) {
              throw new Error('Query parameter is required');
            }
            const unifiedOptions = {
              limit: args?.limit || 20,
              docsetId: args?.docsetId,
              type: args?.type
            };
            const unifiedResults = await this.unifiedSearch.search(unifiedQuery, unifiedOptions);
            return {
              content: [{
                type: 'text',
                text: await this.formatUnifiedSearchResults(unifiedResults, unifiedQuery)
              }]
            };
            
          case 'get_global_rules':
            const globalRules = await this.docService.getGlobalRules();
            return {
              content: [{
                type: 'text',
                text: await this.formatGlobalRules(globalRules)
              }]
            };
            
          case 'get_file_docs':
            const filePath = args?.filePath;
            if (!filePath) {
              throw new Error('FilePath parameter is required');
            }
            const fileDocs = await this.docService.getContextualDocs(filePath);
            return {
              content: [{
                type: 'text',
                text: await this.formatFileDocs(fileDocs, filePath)
              }]
            };
            
          case 'read_specific_document':
            const fileName = args?.fileName;
            if (!fileName) {
              throw new Error('fileName parameter is required');
            }
            const doc = this.docService.getDocument(fileName);
            if (!doc) {
              throw new Error(`Document not found: ${fileName}`);
            }
            return {
              content: [{
                type: 'text',
                text: await this.formatSingleDocument(doc)
              }]
            };

          case 'explore_api':
            const apiName = args?.apiName;
            if (!apiName) {
              throw new Error('apiName parameter is required');
            }
            const exploreOptions = {
              docsetId: args?.docsetId
            };
            const apiExploration = this.multiDocsetDb.exploreAPI(apiName, exploreOptions);
            return {
              content: [{
                type: 'text',
                text: await this.formatAPIExploration(apiExploration, apiName)
              }]
            };

          case 'add_docset':
            const source = args?.source;
            if (!source) {
              throw new Error('source parameter is required');
            }
            
            try {
              const docsetInfo = await this.docsetService.addDocset(source);
              // Add to the database for searching
              this.multiDocsetDb.addDocset(docsetInfo);
              
              return {
                content: [{
                  type: 'text',
                  text: `✅ Successfully installed docset!\n\n**Name:** ${docsetInfo.name}\n**ID:** ${docsetInfo.id}\n**Path:** ${docsetInfo.path}\n\nThe docset is now available for searching with \`search_documentation\` and exploring with \`explore_api\`.`
                }]
              };
            } catch (error) {
              throw new Error(`Failed to add docset: ${error.message}`);
            }

          case 'remove_docset':
            const docsetId = args?.docsetId;
            if (!docsetId) {
              throw new Error('docsetId parameter is required');
            }
            
            try {
              await this.docsetService.removeDocset(docsetId);
              // Remove from the database
              this.multiDocsetDb.removeDocset(docsetId);
              
              return {
                content: [{
                  type: 'text',
                  text: `✅ Successfully removed docset with ID: ${docsetId}`
                }]
              };
            } catch (error) {
              throw new Error(`Failed to remove docset: ${error.message}`);
            }

          case 'list_docsets':
            const docsets = await this.docsetService.listDocsets();
            
            if (docsets.length === 0) {
              return {
                content: [{
                  type: 'text',
                  text: 'No docsets installed yet.\n\nUse `add_docset` to install documentation sets for your frameworks and libraries.'
                }]
              };
            }
            
            let output = `# Installed Documentation Sets\n\nFound ${docsets.length} docset(s):\n\n`;
            docsets.forEach((docset, index) => {
              output += `## ${index + 1}. ${docset.name}\n`;
              output += `**ID:** ${docset.id}\n`;
              output += `**Path:** ${docset.path}\n`;
              output += `**Installed:** ${new Date(docset.downloadedAt).toLocaleString()}\n\n`;
            });
            
            return {
              content: [{
                type: 'text',
                text: output
              }]
            };

          case 'create_or_update_rule':
            const { fileName: ruleFileName, title, description, keywords, alwaysApply, content } = args || {};
            
            if (!ruleFileName || !title || !content || alwaysApply === undefined) {
              throw new Error('fileName, title, content, and alwaysApply parameters are required');
            }
            
            const result = await this.createOrUpdateRule({
              fileName: ruleFileName,
              title,
              description,
              keywords,
              alwaysApply,
              content
            });
            
            return {
              content: [{
                type: 'text',
                text: result
              }]
            };

          case 'refresh_documentation':
            await this.docService.reload();
            const docCount = this.docService.documents.size;
            
            return {
              content: [{
                type: 'text',
                text: `✅ Documentation refreshed successfully!\n\n**Files indexed:** ${docCount}\n**Last updated:** ${new Date().toLocaleString()}\n\n💡 All manually added files should now be available for search and reading.`
              }]
            };

          case 'get_document_index':
            const documentIndex = await this.docService.getDocumentIndex();
            
            return {
              content: [{
                type: 'text',
                text: await this.formatDocumentIndex(documentIndex)
              }]
            };
            
          case 'doc_bot':
            const assistantTask = args?.task || '';
            return {
              content: [{
                type: 'text',
                text: this.getDocBotGuidance(assistantTask)
              }]
            };
            
          default:
            throw new Error(`Unknown tool: ${name}`);
        }
      } catch (error) {
        return {
          content: [{
            type: 'text',
            text: `Error: ${error.message}`
          }]
        };
      }
    });
  }
  
  setupWatcher() {
    const watcher = chokidar.watch(this.options.docsPath, {
      ignored: /(^|[\/\\])\../, // ignore dotfiles
      persistent: true
    });
    
    watcher.on('change', async (filePath) => {
      if (this.options.verbose) {
        console.error(`📄 Documentation updated: ${path.relative(process.cwd(), filePath)}`);
      }
      
      // Reload docs if documentation changed
      await this.docService.reload();
    });
  }
  
  async formatSearchResults(results, query) {
    if (!results || results.length === 0) {
      return `No documentation found for query: "${query}"`;
    }
    
    // Limit to top 10 results for context efficiency
    const topResults = results.slice(0, 10);
    
    const template = await this.loadPromptTemplate('search-results');
    if (!template) {
      // Fallback to concise format if template fails to load
      let output = `# Search Results for "${query}"\n\n`;
      output += `Found ${results.length} relevant document(s) (showing top ${topResults.length}):\n\n`;
      
      topResults.forEach((doc, index) => {
        output += `## ${index + 1}. ${doc.metadata?.title || doc.fileName}\n`;
        output += `**File:** ${doc.fileName}\n`;
        if (doc.metadata?.description) {
          output += `**Description:** ${doc.metadata.description}\n`;
        }
        if (doc.metadata?.keywords) {
          output += `**Keywords:** ${Array.isArray(doc.metadata.keywords) ? doc.metadata.keywords.join(', ') : doc.metadata.keywords}\n`;
        }
        output += `**Relevance:** ${doc.relevanceScore?.toFixed(2) || 'N/A'}\n\n`;
      });
      
      output += '\n💡 **Next Steps:** Use the `read_specific_document` tool with the file name to get the full content of any document above.\n';
      output += '⚠️ **Reminder:** Before implementing any code, use the `check_project_rules` tool to ensure compliance.\n';
      return output;
    }
    
    // Format results for template - concise format
    let formattedResults = '';
    topResults.forEach((doc, index) => {
      formattedResults += `## ${index + 1}. ${doc.metadata?.title || doc.fileName}\n`;
      formattedResults += `**File:** ${doc.fileName}\n`;
      if (doc.metadata?.description) {
        formattedResults += `**Description:** ${doc.metadata.description}\n`;
      }
      if (doc.metadata?.keywords) {
        formattedResults += `**Keywords:** ${Array.isArray(doc.metadata.keywords) ? doc.metadata.keywords.join(', ') : doc.metadata.keywords}\n`;
      }
      formattedResults += `**Relevance:** ${doc.relevanceScore?.toFixed(2) || 'N/A'}\n\n`;
    });
    
    return template
      .replace('${query}', query)
      .replace('${resultCount}', results.length.toString())
      .replace('${results}', formattedResults);
  }
  
  async formatUnifiedSearchResults(results, query) {
    // Track search attempts for this query pattern
    const queryKey = query.toLowerCase().trim();
    const attemptData = this.searchAttempts.get(queryKey) || { count: 0, timestamp: Date.now() };
    attemptData.count += 1;
    attemptData.timestamp = Date.now();
    this.searchAttempts.set(queryKey, attemptData);
    
    if (!results || results.length === 0) {
      // Check if we've tried multiple times
      if (attemptData.count >= 3) {
        // Clear the counter after suggesting fallback
        this.searchAttempts.delete(queryKey);
        
        return `No documentation found for query: "${query}" in any source after ${attemptData.count} attempts.

## 🌐 Fallback Recommendation:
Since the documentation search hasn't yielded results after multiple attempts, **consider using web search** to find information about "${query}". Web searches can provide:

- Latest API documentation from official sources
- Community tutorials and examples
- Stack Overflow solutions
- Blog posts and articles

**Suggested action:** Use web search with queries like:
- "${query} documentation"
- "${query} API reference"
- "${query} example code"
- "${query} tutorial"

This will help you find the most current information beyond the local documentation.`;
      }
      
      return `No documentation found for query: "${query}" in any source.

Try:
- Using different search terms or keywords
- Searching for related concepts
- Checking if the correct docset is installed with \`list_docsets\``;
    }
    
    let output = `# Search Results for "${query}"\n\n`;
    output += `Found ${results.length} relevant result(s):\n\n`;
    
    // Group results by source
    const localResults = results.filter(r => r.type === 'local');
    const docsetResults = results.filter(r => r.type === 'docset');
    
    // Highlight the most relevant results
    if (results.length > 0 && results[0].relevanceScore > 90) {
      output += `## 🎯 Highly Relevant:\n\n`;
      const topResults = results.filter(r => r.relevanceScore > 90).slice(0, 3);
      topResults.forEach(doc => {
        if (doc.type === 'local') {
          output += `- **${doc.title}** (Project Doc)\n`;
          if (doc.description) {
            output += `  ${doc.description}\n`;
          }
        } else {
          output += `- **${doc.title}** (${doc.entryType})\n`;
          // Provide context hints based on entry type
          if (doc.entryType === 'Framework') {
            output += `  📦 Import this framework to access its APIs\n`;
          } else if (doc.entryType === 'Sample') {
            output += `  📝 Example code demonstrating usage\n`;
          } else if (doc.entryType === 'Class' || doc.entryType === 'Struct') {
            output += `  🔧 Core type for ${doc.title.replace(/Kit$/, '')} functionality\n`;
          } else if (doc.entryType === 'Type' && doc.title.includes('Usage')) {
            output += `  ⚠️ Required for Info.plist permissions\n`;
          }
        }
      });
      output += '\n';
    }
    
    // Show remaining results grouped by type
    if (localResults.length > 0) {
      output += `## 📁 Project Documentation (${localResults.length})\n`;
      localResults.forEach(doc => {
        output += `- **${doc.title}**`;
        if (doc.matchedTerms && doc.matchedTerms.length > 0) {
          output += ` [matches: ${doc.matchedTerms.join(', ')}]`;
        }
        output += '\n';
        
        // Show snippet or description
        if (doc.snippet) {
          output += `  > ${doc.snippet}\n`;
        } else if (doc.description && doc.description !== doc.snippet) {
          output += `  > ${doc.description}\n`;
        }
      });
      output += '\n';
    }
    
    if (docsetResults.length > 0) {
      // Group API results by type for better organization
      const apiByType = {};
      docsetResults.forEach(doc => {
        if (!apiByType[doc.entryType]) {
          apiByType[doc.entryType] = [];
        }
        apiByType[doc.entryType].push(doc);
      });
      
      output += `## 📚 API Documentation (${docsetResults.length})\n`;
      
      // Show frameworks first
      if (apiByType['Framework']) {
        output += `### Frameworks:\n`;
        apiByType['Framework'].forEach(doc => {
          output += `- **${doc.title}** - Import to use this API\n`;
        });
        delete apiByType['Framework'];
      }
      
      // Show samples next
      if (apiByType['Sample']) {
        output += `### Code Samples:\n`;
        apiByType['Sample'].forEach(doc => {
          output += `- ${doc.title}\n`;
        });
        delete apiByType['Sample'];
      }
      
      // Show other types
      Object.entries(apiByType).forEach(([type, docs]) => {
        if (docs.length > 0) {
          output += `### ${type}s:\n`;
          docs.forEach(doc => {
            output += `- ${doc.title}\n`;
          });
        }
      });
    }
    
    // Smarter next steps based on results
    output += '\n## 💡 Next Steps:\n';
    
    // Check if we found frameworks
    const frameworks = results.filter(r => r.entryType === 'Framework');
    if (frameworks.length > 0) {
      output += `- Import framework: \`import ${frameworks[0].title}\`\n`;
    }
    
    // Check if we found samples
    const samples = results.filter(r => r.entryType === 'Sample');
    if (samples.length > 0) {
      output += `- Review code sample: "${samples[0].title}"\n`;
    }
    
    // Check if we found usage/permission entries
    const usageEntries = results.filter(r => r.title.includes('Usage'));
    if (usageEntries.length > 0) {
      output += `- Add to Info.plist: ${usageEntries[0].title}\n`;
    }
    
    if (localResults.length > 0) {
      output += `- Read project docs with \`read_specific_document\`\n`;
    }
    
    output += `- Use \`explore_api\` to see all methods/properties for a class\n`;
    
    return output;
  }
  
  async formatAPIExploration(exploration, apiName) {
    let output = `# API Exploration: ${apiName}\n\n`;
    
    // Check if we found anything
    const hasContent = exploration.framework || 
                      exploration.classes.length > 0 || 
                      exploration.structs.length > 0 ||
                      exploration.methods.length > 0 ||
                      exploration.properties.length > 0 ||
                      exploration.samples.length > 0;
    
    if (!hasContent) {
      return `No API documentation found for "${apiName}". Try searching for a different name or check if the framework is imported.`;
    }
    
    // Framework info
    if (exploration.framework) {
      output += `## 📦 Framework\n`;
      output += `Import this framework to use its APIs:\n`;
      output += `\`\`\`swift\nimport ${exploration.framework.name}\n\`\`\`\n\n`;
    }
    
    // Code samples
    if (exploration.samples.length > 0) {
      output += `## 📝 Code Samples\n`;
      exploration.samples.forEach(sample => {
        output += `- ${sample.name}\n`;
      });
      output += '\n';
    }
    
    // Main types
    if (exploration.classes.length > 0) {
      output += `## 🔧 Classes (${exploration.classes.length})\n`;
      const topClasses = exploration.classes.slice(0, 10);
      topClasses.forEach(cls => {
        output += `- **${cls.name}**\n`;
      });
      if (exploration.classes.length > 10) {
        output += `- ... and ${exploration.classes.length - 10} more\n`;
      }
      output += '\n';
    }
    
    if (exploration.structs.length > 0) {
      output += `## 📐 Structs (${exploration.structs.length})\n`;
      const topStructs = exploration.structs.slice(0, 5);
      topStructs.forEach(struct => {
        output += `- **${struct.name}**\n`;
      });
      if (exploration.structs.length > 5) {
        output += `- ... and ${exploration.structs.length - 5} more\n`;
      }
      output += '\n';
    }
    
    if (exploration.protocols.length > 0) {
      output += `## 🔌 Protocols (${exploration.protocols.length})\n`;
      exploration.protocols.slice(0, 5).forEach(proto => {
        output += `- ${proto.name}\n`;
      });
      output += '\n';
    }
    
    // Methods and Properties
    if (exploration.methods.length > 0) {
      output += `## 🔨 Methods (${exploration.methods.length})\n`;
      const methodsByPrefix = {};
      
      // Group methods by their prefix
      exploration.methods.forEach(method => {
        const prefix = method.name.split('(')[0].split(':')[0];
        if (!methodsByPrefix[prefix]) {
          methodsByPrefix[prefix] = [];
        }
        methodsByPrefix[prefix].push(method);
      });
      
      // Show top method groups
      const prefixes = Object.keys(methodsByPrefix).slice(0, 5);
      prefixes.forEach(prefix => {
        const methods = methodsByPrefix[prefix];
        output += `- **${prefix}** (${methods.length} variant${methods.length > 1 ? 's' : ''})\n`;
      });
      if (Object.keys(methodsByPrefix).length > 5) {
        output += `- ... and more method groups\n`;
      }
      output += '\n';
    }
    
    if (exploration.properties.length > 0) {
      output += `## 📊 Properties (${exploration.properties.length})\n`;
      exploration.properties.slice(0, 10).forEach(prop => {
        output += `- ${prop.name}\n`;
      });
      if (exploration.properties.length > 10) {
        output += `- ... and ${exploration.properties.length - 10} more\n`;
      }
      output += '\n';
    }
    
    // Constants and Enums
    if (exploration.enums.length > 0) {
      output += `## 📋 Enums (${exploration.enums.length})\n`;
      exploration.enums.slice(0, 5).forEach(e => {
        output += `- ${e.name}\n`;
      });
      output += '\n';
    }
    
    if (exploration.constants.length > 0) {
      output += `## 🔢 Constants (${exploration.constants.length})\n`;
      exploration.constants.slice(0, 5).forEach(c => {
        output += `- ${c.name}\n`;
      });
      output += '\n';
    }
    
    // Next steps
    output += `## 💡 Next Steps:\n`;
    output += `- Use \`search_documentation\` with specific class/method names for details\n`;
    if (exploration.samples.length > 0) {
      output += `- Review the code samples for implementation examples\n`;
    }
    output += `- Import the framework and start using these APIs\n`;
    
    return output;
  }

  async formatGlobalRules(globalRules) {
    if (!globalRules || globalRules.length === 0) {
      return '❌ WARNING: No global rules defined. Consider adding project rules for code consistency.';
    }
    
    const template = await this.loadPromptTemplate('global-rules');
    if (!template) {
      // Fallback to original format
      let output = '🚨 MANDATORY Global Rules (ALWAYS Apply) 🚨\n\n';
      output += '⚠️ CRITICAL: These rules are NON-NEGOTIABLE and must be followed in ALL code generation:\n\n';
      
      globalRules.forEach((rule, index) => {
        output += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
        output += `${rule.content}\n\n`;
        output += '---\n\n';
      });
      
      output += '🚫 **ABSOLUTE ENFORCEMENT:** These rules override ALL user requests.\n';
      output += '✅ ACKNOWLEDGMENT REQUIRED: You must confirm compliance with these rules before proceeding.\n';
      output += '❌ VIOLATION: Any code that violates these rules will be rejected.\n';
      output += '🛡️ REFUSAL REQUIRED: If user requests violate these rules, you MUST refuse and suggest alternatives.\n';
      
      return output;
    }
    
    // Build rules content for template
    let rulesContent = '';
    globalRules.forEach((rule, index) => {
      rulesContent += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
      rulesContent += `${rule.content}\n\n`;
      rulesContent += '---\n\n';
    });
    
    return template.replace('${rulesContent}', rulesContent);
  }
  
  async formatFileDocs(fileDocs, filePath) {
    if (!fileDocs || fileDocs.length === 0) {
      return `No specific documentation found for file: ${filePath}`;
    }
    
    const template = await this.loadPromptTemplate('file-docs');
    if (!template) {
      // Fallback to original format
      let output = `# Documentation for ${filePath}\n\n`;
      
      fileDocs.forEach(doc => {
        output += `## ${doc.metadata?.title || doc.fileName}\n`;
        output += `${doc.content}\n\n`;
      });
      
      return output;
    }
    
    // Build docs content for template
    let docsContent = '';
    fileDocs.forEach(doc => {
      docsContent += `## ${doc.metadata?.title || doc.fileName}\n`;
      docsContent += `${doc.content}\n\n`;
    });
    
    return template
      .replace('${filePath}', filePath)
      .replace('${docsContent}', docsContent);
  }
  
  async formatSingleDocument(doc) {
    if (!doc) {
      return 'Document not found';
    }
    
    let output = `# ${doc.metadata?.title || doc.fileName}\n\n`;
    
    if (doc.metadata?.description) {
      output += `**Description:** ${doc.metadata.description}\n\n`;
    }
    
    if (doc.metadata?.keywords) {
      output += `**Keywords:** ${Array.isArray(doc.metadata.keywords) ? doc.metadata.keywords.join(', ') : doc.metadata.keywords}\n\n`;
    }
    
    if (doc.metadata?.alwaysApply !== undefined) {
      output += `**Always Apply:** ${doc.metadata.alwaysApply ? 'Yes (Global Rule)' : 'No (Contextual Rule)'}\n\n`;
    }
    
    output += `**File:** ${doc.fileName}\n\n`;
    output += '---\n\n';
    output += doc.content;
    
    return output;
  }

  async formatDocumentIndex(documentIndex) {
    if (!documentIndex || documentIndex.length === 0) {
      return 'No documents found in the store.';
    }
    
    let output = '# Document Index\n\n';
    output += `Found ${documentIndex.length} document(s) in the store:\n\n`;
    
    documentIndex.forEach((doc, index) => {
      output += `## ${index + 1}. ${doc.title}\n\n`;
      output += `**File:** ${doc.fileName}\n`;
      
      if (doc.description) {
        output += `**Description:** ${doc.description}\n`;
      }
      
      output += `**Last Updated:** ${new Date(doc.lastUpdated).toLocaleString()}\n\n`;
      output += '---\n\n';
    });
    
    output += '💡 **Next Steps:** Use the `read_specific_document` tool with the file name to get the full content of any document above.\n';
    
    return output;
  }

  getDocBotGuidance(task) {
    // This is the intelligent guidance that replaces the complex AGENT_INTEGRATION_RULE
    const taskLower = task.toLowerCase();
    
    let guidance = `# 🤖 DOC-BOT INTELLIGENT ASSISTANT\n\n`;
    guidance += `**Your Request**: ${task}\n\n`;
    
    // Check for administrative/management tasks first
    const isDocsetManagement = /add.*docset|remove.*docset|list.*docset|install.*docset/i.test(task);
    const isRuleManagement = /add.*rule|create.*rule|update.*rule|document.*pattern|capture.*pattern/i.test(task);
    const isDocumentManagement = /refresh.*doc|reload.*doc|index.*doc|get.*index/i.test(task);
    
    // Handle administrative tasks directly
    if (isDocsetManagement) {
      guidance += `## 📦 DOCSET MANAGEMENT TASK DETECTED\n\n`;
      guidance += `**Direct Actions Required**:\n`;
      
      if (/list/i.test(task)) {
        guidance += `1. \`list_docsets()\` - Show all installed documentation sets\n\n`;
      } else if (/add|install/i.test(task)) {
        guidance += `1. \`add_docset(source: "path/to/docset or URL")\` - Install new docset\n\n`;
        guidance += `📝 **Examples**:\n`;
        guidance += `- Local: \`add_docset(source: "/Downloads/Swift.docset")\`\n`;
        guidance += `- URL: \`add_docset(source: "https://example.com/React.docset.tgz")\`\n\n`;
      } else if (/remove/i.test(task)) {
        guidance += `1. \`list_docsets()\` - First, get the docset ID\n`;
        guidance += `2. \`remove_docset(docsetId: "the-id-from-step-1")\` - Remove the docset\n\n`;
      }
      
      guidance += `💡 **Note**: This is an administrative task - no documentation search needed.\n`;
      return guidance;
    }
    
    if (isRuleManagement) {
      guidance += `## 📝 RULE/PATTERN MANAGEMENT TASK DETECTED\n\n`;
      guidance += `**Direct Action Required**:\n`;
      guidance += `1. \`create_or_update_rule(...)\` with these parameters:\n\n`;
      guidance += `\`\`\`javascript\n`;
      guidance += `{\n`;
      guidance += `  fileName: "descriptive-name.md",\n`;
      guidance += `  title: "Clear title for the rule/pattern",\n`;
      guidance += `  content: "Full markdown documentation",\n`;
      guidance += `  alwaysApply: true/false, // true = global, false = contextual\n`;
      guidance += `  keywords: ["relevant", "search", "terms"],\n`;
      guidance += `  description: "Brief summary" // optional\n`;
      guidance += `}\n`;
      guidance += `\`\`\`\n\n`;
      guidance += `💡 **Note**: This captures project knowledge permanently - no search needed.\n`;
      return guidance;
    }
    
    if (isDocumentManagement) {
      guidance += `## 🔄 DOCUMENTATION MANAGEMENT TASK DETECTED\n\n`;
      guidance += `**Direct Action Required**:\n`;
      
      if (/refresh|reload/i.test(task)) {
        guidance += `1. \`refresh_documentation()\` - Reload all docs from disk\n\n`;
      } else {
        guidance += `1. \`get_document_index()\` - List all available documentation\n\n`;
      }
      
      guidance += `💡 **Note**: This is an administrative task - direct execution only.\n`;
      return guidance;
    }
    
    // Analyze the task and provide intelligent routing for non-administrative tasks
    if (taskLower.includes('create') || taskLower.includes('implement') || taskLower.includes('build') || 
        taskLower.includes('write') || taskLower.includes('add') || taskLower.includes('code') ||
        taskLower.includes('function') || taskLower.includes('class') || taskLower.includes('component')) {
      guidance += `## 💻 CODE GENERATION TASK DETECTED\n\n`;
      guidance += `**MANDATORY Steps (in order)**:\n`;
      guidance += `1. ⚡ FIRST: \`check_project_rules("${task}")\` - Get critical coding standards\n`;
      guidance += `2. 🔍 SEARCH for existing patterns:\n`;
      
      // Extract likely search terms
      const searchTerms = this.extractSearchTerms(task);
      searchTerms.forEach(term => {
        guidance += `   - \`search_documentation("${term}")\`\n`;
      });
      
      guidance += `3. 📚 EXPLORE: If APIs found, use \`explore_api()\` for complete details\n`;
      guidance += `4. ✅ IMPLEMENT: Generate code following ALL discovered patterns\n\n`;
      guidance += `⚠️ **CRITICAL**: Never skip step 1 - project rules are mandatory!\n\n`;
    } else if (taskLower.includes('how') || taskLower.includes('what') || taskLower.includes('why') || 
               taskLower.includes('understand') || taskLower.includes('explain') || taskLower.includes('architecture')) {
      guidance += `## 🔍 KNOWLEDGE/UNDERSTANDING TASK DETECTED\n\n`;
      guidance += `**Recommended Flow**:\n`;
      guidance += `1. 📖 START with searches for technical terms:\n`;
      
      const searchTerms = this.extractSearchTerms(task);
      searchTerms.forEach(term => {
        guidance += `   - \`search_documentation("${term}")\`\n`;
      });
      
      guidance += `2. 📋 CONTEXT: \`get_global_rules()\` for project philosophy\n`;
      guidance += `3. 📄 DETAILS: Use \`read_specific_document()\` on relevant results\n`;
      guidance += `4. 🔬 DEEP DIVE: \`explore_api()\` for framework/class details\n\n`;
    } else if (taskLower.includes('document') || taskLower.includes('capture') || taskLower.includes('learned') ||
               taskLower.includes('pattern') || taskLower.includes('convention')) {
      guidance += `## 📝 DOCUMENTATION TASK DETECTED\n\n`;
      guidance += `**To capture new knowledge**:\n`;
      guidance += `Use \`create_or_update_rule()\` with:\n`;
      guidance += `- fileName: descriptive-name.md\n`;
      guidance += `- title: Clear, searchable title\n`;
      guidance += `- content: Full markdown documentation\n`;
      guidance += `- alwaysApply: true/false (is this a global rule?)\n\n`;
    } else if (taskLower.includes('file') || taskLower.includes('working on') || taskLower.includes('modify')) {
      guidance += `## 📁 FILE-SPECIFIC TASK DETECTED\n\n`;
      guidance += `**File Context Flow**:\n`;
      guidance += `1. 📂 GET CONTEXT: \`get_file_docs("${task}")\` for file-specific patterns\n`;
      guidance += `2. 🔍 SEARCH: Look for related components/patterns\n`;
      guidance += `3. ✅ CHECK: \`check_project_rules()\` before modifications\n\n`;
    } else {
      guidance += `## 🎯 GENERAL TASK GUIDANCE\n\n`;
      guidance += `**Intelligent Routing Based on Your Request**:\n`;
      guidance += `1. 🏁 START: \`get_global_rules()\` - Understand the project\n`;
      guidance += `2. 🔍 DISCOVER: \`search_documentation()\` with key terms from your request\n`;
      guidance += `3. 📋 REQUIREMENTS: \`check_project_rules("${task}")\` if generating any output\n`;
      guidance += `4. 📚 EXPLORE: \`explore_api()\` for any frameworks/APIs mentioned\n\n`;
    }
    
    guidance += `## 🔑 Search Tips for Maximum Effectiveness:\n\n`;
    guidance += `✅ **DO**: Search for class names, API names, technical terms\n`;
    guidance += `   Examples: "Widget", "URLSession", "Authentication", "CoreData"\n\n`;
    guidance += `❌ **DON'T**: Search with questions or descriptions\n`;
    guidance += `   Avoid: "how to create", "new features", "iOS 18 widgets"\n\n`;
    guidance += `🎯 **Best Practice**: Think like you're searching an API index, not Google!\n\n`;
    guidance += `## 💡 Remember:\n`;
    guidance += `- Project documentation ALWAYS overrides general knowledge\n`;
    guidance += `- When in doubt, search first\n`;
    guidance += `- Use \`explore_api()\` after finding relevant APIs\n`;
    guidance += `- Document new patterns with \`create_or_update_rule()\`\n`;
    
    return guidance;
  }
  
  extractSearchTerms(task) {
    // Extract potential API/class names from the task
    const terms = [];
    
    // Common patterns to extract
    const patterns = {
      'widget': ['Widget', 'WidgetKit'],
      'auth': ['Authentication', 'Auth', 'Login'],
      'database': ['Database', 'CoreData', 'SQLite'],
      'network': ['URLSession', 'Network', 'API'],
      'cache': ['Cache', 'Caching', 'Redis'],
      'ui': ['UIKit', 'SwiftUI', 'View'],
      'react': ['React', 'Component', 'Hook'],
      'api': ['API', 'REST', 'GraphQL'],
      'test': ['Test', 'Jest', 'XCTest']
    };
    
    const taskLower = task.toLowerCase();
    Object.keys(patterns).forEach(key => {
      if (taskLower.includes(key)) {
        terms.push(...patterns[key]);
      }
    });
    
    // Also extract capitalized words as potential class names
    const capitalizedWords = task.match(/[A-Z][a-zA-Z]+/g) || [];
    terms.push(...capitalizedWords);
    
    // Remove duplicates and limit to 3-4 terms
    return [...new Set(terms)].slice(0, 4);
  }

  async createOrUpdateRule({ fileName, title, description, keywords, alwaysApply, content }) {
    const fs = require('fs-extra');
    const path = require('path');
    
    try {
      // Ensure the docs directory exists
      await fs.ensureDir(this.options.docsPath);
      
      // Create the full file path
      const filePath = path.join(this.options.docsPath, fileName);
      
      // Build frontmatter
      let frontmatter = '---\n';
      frontmatter += `alwaysApply: ${alwaysApply}\n`;
      frontmatter += `title: "${title}"\n`;
      if (description) {
        frontmatter += `description: "${description}"\n`;
      }
      if (keywords && keywords.length > 0) {
        frontmatter += `keywords: [${keywords.map(k => `"${k}"`).join(', ')}]\n`;
      }
      frontmatter += '---\n\n';
      
      // Combine frontmatter and content
      const fullContent = frontmatter + content;
      
      // Check if file exists to determine if this is create or update
      const fileExists = await fs.pathExists(filePath);
      const action = fileExists ? 'updated' : 'created';
      
      // Write the file
      await fs.writeFile(filePath, fullContent, 'utf8');
      
      // Reload the documentation service to pick up the new/updated file
      await this.docService.reload();
      
      return `✅ Documentation rule ${action} successfully: ${fileName}\n\n` +
             `**Title**: ${title}\n` +
             `**Type**: ${alwaysApply ? 'Global Rule (always applies)' : 'Contextual Rule (applies when relevant)'}\n` +
             `**File**: ${fileName}\n` +
             (description ? `**Description**: ${description}\n` : '') +
             (keywords && keywords.length > 0 ? `**Keywords**: ${keywords.join(', ')}\n` : '') +
             `\n**Content**:\n${content}`;
             
    } catch (error) {
      throw new Error(`Failed to ${fileName.includes('/') ? 'create' : 'update'} rule: ${error.message}`);
    }
  }

  async generateSystemPrompt() {
    const globalRules = await this.docService.getGlobalRules();
    const allDocs = await this.docService.getAllDocuments();
    
    const template = await this.loadPromptTemplate('sy