mcp-orchestrator
Version:
MCP Orchestrator - Discover and install MCPs with automatic OAuth support. Uses Claude CLI for OAuth MCPs (Canva, Asana, etc). 34 trusted MCPs from Claude Partners.
276 lines (275 loc) • 11.5 kB
JavaScript
/**
* Build Intelligent MCP Index
* This script crawls, indexes, and creates a searchable vector database of ALL MCPs
* Run this to build your orchestrator's brain!
*/
import { MCPMegaCrawler } from './mcp-mega-crawler.js';
import { VectorDiscoveryEngine } from './vector-discovery-prototype.js';
import * as fs from 'fs';
import * as path from 'path';
import * as cron from 'node-cron';
class IntelligentMCPOrchestrator {
crawler;
discoveryEngine;
indexPath = './data/mcp-intelligent-index.json';
constructor() {
this.crawler = new MCPMegaCrawler();
this.discoveryEngine = new VectorDiscoveryEngine(this.indexPath);
}
/**
* Build the intelligent index from scratch
*/
async buildIndex() {
console.log('🧠 Building Intelligent MCP Index...\n');
console.log('This will:');
console.log(' 1. Crawl npm, PyPI, GitHub for ALL MCPs');
console.log(' 2. Generate semantic embeddings');
console.log(' 3. Create searchable vector index');
console.log(' 4. Enable intelligent discovery\n');
// Step 1: Crawl everything
console.log('Step 1/4: Crawling all sources...');
const allMCPs = await this.crawler.crawlAll();
console.log(`✅ Found ${allMCPs.length} unique MCPs!\n`);
// Step 2: Initialize vector engine
console.log('Step 2/4: Initializing vector engine...');
await this.discoveryEngine.initialize();
// Step 3: Add all MCPs to vector index
console.log('Step 3/4: Building vector index...');
for (const mcp of allMCPs) {
await this.discoveryEngine.addMCP({
id: mcp.id,
name: mcp.name,
description: mcp.description,
packageName: mcp.packageName,
keywords: mcp.keywords,
metadata: {
...mcp.metadata,
category: this.categorize(mcp)
}
});
}
// Step 4: Save and verify
console.log('Step 4/4: Saving and verifying...');
const stats = this.discoveryEngine.getStats();
console.log('\n✅ Intelligent Index Built Successfully!');
console.log('📊 Index Statistics:');
console.log(` Total MCPs: ${stats.totalMCPs}`);
console.log(` Categories: ${Object.keys(stats.byCategory).length}`);
console.log(` Runtimes: ${Object.keys(stats.byRuntime).join(', ')}`);
return stats;
}
/**
* Categorize an MCP based on its properties
*/
categorize(mcp) {
const categories = [];
const text = `${mcp.name} ${mcp.description} ${mcp.keywords.join(' ')}`.toLowerCase();
// Category detection rules
const categoryRules = {
'data': ['csv', 'excel', 'data', 'pandas', 'analysis', 'statistics'],
'database': ['sql', 'database', 'postgres', 'mysql', 'sqlite', 'mongo'],
'api': ['api', 'rest', 'graphql', 'webhook', 'http'],
'automation': ['automate', 'browser', 'puppeteer', 'selenium', 'scrape'],
'storage': ['file', 'filesystem', 's3', 'storage', 'blob'],
'communication': ['slack', 'discord', 'email', 'sms', 'notification'],
'ai': ['openai', 'claude', 'llm', 'embedding', 'vector'],
'devops': ['docker', 'kubernetes', 'deploy', 'ci', 'cd'],
'cloud': ['aws', 'azure', 'gcp', 'cloud', 'serverless'],
'version-control': ['git', 'github', 'gitlab', 'commit', 'merge']
};
for (const [category, keywords] of Object.entries(categoryRules)) {
if (keywords.some(keyword => text.includes(keyword))) {
categories.push(category);
}
}
return categories.length > 0 ? categories : ['general'];
}
/**
* Test the intelligent discovery
*/
async testDiscovery() {
console.log('\n🧪 Testing Intelligent Discovery...\n');
const testQueries = [
"I need to analyze a large CSV file with pandas-like operations",
"How can I automate browser testing and take screenshots?",
"I want to work with Notion API to manage my workspace",
"Help me query a local SQLite database",
"I need to scrape data from websites",
"Connect to Slack and send notifications",
"Work with GitHub issues and pull requests",
"Process and transform JSON data",
"Generate PDFs from web pages",
"Monitor application errors and performance"
];
for (const query of testQueries) {
console.log(`\n📝 Query: "${query}"`);
console.log('─'.repeat(60));
const results = await this.discoveryEngine.discover(query, 5);
if (results.length === 0) {
console.log('❌ No relevant MCPs found');
continue;
}
results.forEach((result, i) => {
console.log(`\n${i + 1}. ${result.mcp.name}`);
console.log(` Score: ${(result.score * 100).toFixed(1)}%`);
console.log(` Package: ${result.mcp.packageName}`);
console.log(` Description: ${result.mcp.description.substring(0, 100)}...`);
console.log(` Reason: ${result.reason}`);
// Installation command
const runtime = result.mcp.metadata.runtime;
if (runtime === 'node') {
console.log(` Install: npm install -g ${result.mcp.packageName}`);
}
else if (runtime === 'python') {
console.log(` Install: pip install ${result.mcp.packageName}`);
}
});
}
}
/**
* Schedule automatic index updates
*/
scheduleUpdates() {
console.log('📅 Scheduling automatic index updates...');
// Run every Sunday at 2 AM
cron.schedule('0 2 * * 0', async () => {
console.log('🔄 Running scheduled index update...');
await this.updateIndex();
});
console.log('✅ Scheduled weekly updates (Sundays at 2 AM)');
}
/**
* Update the index with new MCPs
*/
async updateIndex() {
console.log('🔄 Updating MCP index...');
const beforeStats = this.discoveryEngine.getStats();
console.log(` Current MCPs: ${beforeStats.totalMCPs}`);
// Crawl for new MCPs
const newMCPs = await this.crawler.crawlAll();
console.log(` Found ${newMCPs.length} total MCPs`);
// Update vector index
await this.discoveryEngine.updateIndex();
const afterStats = this.discoveryEngine.getStats();
const newCount = afterStats.totalMCPs - beforeStats.totalMCPs;
if (newCount > 0) {
console.log(`✅ Added ${newCount} new MCPs to index`);
}
else {
console.log('✅ Index is up to date');
}
}
/**
* Export index for distribution
*/
async exportIndex(outputPath) {
const exportPath = outputPath || `./dist/mcp-index-${Date.now()}.json`;
console.log(`📦 Exporting index to ${exportPath}...`);
// Read the current index
const indexData = fs.readFileSync(this.indexPath, 'utf-8');
const index = JSON.parse(indexData);
// Add metadata
const exportData = {
version: '1.0.0',
created: new Date().toISOString(),
totalMCPs: index.length,
schema: 'vector-index-v1',
index: index
};
// Ensure dist directory exists
const dir = path.dirname(exportPath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
// Write export file
fs.writeFileSync(exportPath, JSON.stringify(exportData));
// Create compressed version
const { exec } = require('child_process');
const compressedPath = exportPath.replace('.json', '.json.gz');
exec(`gzip -c ${exportPath} > ${compressedPath}`, (error) => {
if (!error) {
console.log(`✅ Compressed index: ${compressedPath}`);
}
});
console.log(`✅ Index exported successfully`);
console.log(` Size: ${(fs.statSync(exportPath).size / 1024 / 1024).toFixed(2)} MB`);
}
/**
* Import a pre-built index
*/
async importIndex(importPath) {
console.log(`📥 Importing index from ${importPath}...`);
if (!fs.existsSync(importPath)) {
throw new Error(`Index file not found: ${importPath}`);
}
const importData = JSON.parse(fs.readFileSync(importPath, 'utf-8'));
console.log(` Version: ${importData.version}`);
console.log(` Created: ${importData.created}`);
console.log(` MCPs: ${importData.totalMCPs}`);
// Ensure data directory exists
const dataDir = path.dirname(this.indexPath);
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
}
// Save the index
fs.writeFileSync(this.indexPath, JSON.stringify(importData.index, null, 2));
// Reinitialize discovery engine
await this.discoveryEngine.initialize();
console.log('✅ Index imported successfully');
}
}
// CLI Interface
async function main() {
const orchestrator = new IntelligentMCPOrchestrator();
const args = process.argv.slice(2);
const command = args[0];
console.log('🚀 MCP Intelligent Orchestrator\n');
switch (command) {
case 'build':
await orchestrator.buildIndex();
break;
case 'test':
await orchestrator.testDiscovery();
break;
case 'update':
await orchestrator.updateIndex();
break;
case 'schedule':
orchestrator.scheduleUpdates();
console.log('Scheduler running... Press Ctrl+C to stop');
// Keep process alive
process.stdin.resume();
break;
case 'export':
await orchestrator.exportIndex(args[1]);
break;
case 'import':
if (!args[1]) {
console.error('❌ Please provide import file path');
process.exit(1);
}
await orchestrator.importIndex(args[1]);
break;
case 'stats':
await orchestrator.discoveryEngine.initialize();
console.log(orchestrator.discoveryEngine.getStats());
break;
default:
console.log('Usage: node build-intelligent-index.js <command>\n');
console.log('Commands:');
console.log(' build - Build the intelligent index from scratch');
console.log(' test - Test discovery with sample queries');
console.log(' update - Update index with new MCPs');
console.log(' schedule - Run automatic weekly updates');
console.log(' export - Export index for distribution');
console.log(' import - Import a pre-built index');
console.log(' stats - Show index statistics');
process.exit(0);
}
}
// Run if executed directly
if (import.meta.url === `file://${__filename}`) {
main().catch(console.error);
}
export { IntelligentMCPOrchestrator };