mcp-orchestrator/dist/crawler/official-crawler.js

MCP Orchestrator - Discover and install MCPs with automatic OAuth support. Uses Claude CLI for OAuth MCPs (Canva, Asana, etc). 34 trusted MCPs from Claude Partners.

/**
 * Official MCP Crawler
 * Fetches high-quality MCPs from official sources
 */
import * as https from 'https';
import * as fs from 'fs';
export class OfficialMCPCrawler {
    mcps = [];
    REGISTRY_API_URL = 'https://registry.modelcontextprotocol.io/v0/servers';
    /**
     * Fetch data from the registry API with pagination
     */
    async fetchFromRegistryAPI(limit = 100) {
        const allServers = [];
        let cursor = undefined;
        while (allServers.length < limit) {
            const url = cursor
                ? `${this.REGISTRY_API_URL}?limit=100&cursor=${cursor}`
                : `${this.REGISTRY_API_URL}?limit=100`;
            console.log(`📡 Fetching from: ${url}`);
            const data = await this.httpsGet(url);
            const response = JSON.parse(data);
            // Extract server info from entries
            const serverInfos = response.servers.map(entry => entry.server);
            allServers.push(...serverInfos);
            console.log(`  → Got ${serverInfos.length} servers (total: ${allServers.length})`);
            if (!response.metadata?.nextCursor || allServers.length >= limit) {
                break;
            }
            cursor = response.metadata.nextCursor;
        }
        return allServers.slice(0, limit);
    }
    /**
     * Helper to make HTTPS GET requests
     */
    httpsGet(url) {
        return new Promise((resolve, reject) => {
            https.get(url, (res) => {
                let data = '';
                res.on('data', chunk => data += chunk);
                res.on('end', () => resolve(data));
            }).on('error', reject);
        });
    }
    /**
     * Extract keywords from description
     */
    extractKeywords(description) {
        const stopWords = new Set(['the', 'and', 'for', 'with', 'this', 'that', 'from', 'your', 'can', 'use']);
        const words = description.toLowerCase()
            .replace(/[^\w\s]/g, ' ')
            .split(/\s+/)
            .filter(w => w.length > 3 && !stopWords.has(w));
        return [...new Set(words)].slice(0, 10); // Max 10 unique keywords
    }
    /**
     * Generate ID from server name (use the full qualified name)
     */
    generateId(serverName) {
        // Use the server name as-is since it's already a unique identifier like "ai.mcpcap/mcpcap"
        return serverName
            .replace(/\//g, '-') // Replace / with -
            .toLowerCase();
    }
    /**
     * Transform registry server to our CrawledMCP format
     */
    transformServer(server) {
        // Determine how to connect/install this MCP
        let packageName = '';
        let runtime = 'unknown';
        let command;
        // Check for installable packages (npm, pypi)
        if (server.packages && server.packages.length > 0) {
            const pkg = server.packages.find(p => p.registryType === 'npm' || p.registryType === 'pypi');
            if (pkg) {
                packageName = pkg.identifier;
                runtime = pkg.registryType === 'npm' ? 'node' : 'python';
            }
            else if (server.packages[0]) {
                // Use OCI or other package type
                packageName = server.packages[0].identifier;
                runtime = 'unknown';
            }
        }
        // Check for remote endpoints
        if (server.remotes && server.remotes.length > 0) {
            const remote = server.remotes[0];
            // For remotes, use the URL as the "package" identifier
            if (!packageName) {
                packageName = remote.url;
                command = `remote:${remote.type}:${remote.url}`;
            }
        }
        // Skip if we have no way to connect
        if (!packageName) {
            return null;
        }
        return {
            id: this.generateId(server.name), // Use server name for unique ID
            name: server.name,
            description: server.description || '',
            packageName,
            command,
            keywords: this.extractKeywords(server.description || server.name),
            source: 'official', // From official registry
            metadata: {
                runtime,
                repository: server.repository?.url,
                author: server.author?.name,
                installTime: runtime === 'node' ? '20s' : runtime === 'python' ? '30s' : '5s' // Remotes are instant
            }
        };
    }
    /**
     * Fetch official MCPs from the registry API
     */
    async crawlOfficialRegistry() {
        console.log('🔍 Crawling official MCP registry API...');
        try {
            const servers = await this.fetchFromRegistryAPI(200); // Fetch 200 MCPs
            console.log(`📦 Fetched ${servers.length} servers from registry`);
            const transformed = servers
                .map(s => this.transformServer(s))
                .filter((mcp) => mcp !== null);
            // Deduplicate by ID (keep first occurrence)
            const seenIds = new Set();
            const deduplicated = transformed.filter(mcp => {
                if (seenIds.has(mcp.id)) {
                    return false;
                }
                seenIds.add(mcp.id);
                return true;
            });
            this.mcps = deduplicated;
            console.log(`✅ Successfully transformed ${this.mcps.length} unique MCPs (${transformed.length - deduplicated.length} duplicates removed)`);
            return this.mcps;
        }
        catch (error) {
            console.error('❌ Failed to fetch from registry:', error);
            throw error;
        }
    }
    /**
     * Fetch community MCPs (optional - registry already has community MCPs)
     */
    async crawlCommunityMCPs() {
        console.log('ℹ️  Skipping separate community crawl - registry includes community MCPs');
        return []; // Registry API already includes community servers
    }
    /**
     * Get all crawled MCPs
     */
    getAllMCPs() {
        return this.mcps;
    }
    /**
     * Save crawled MCPs to JSON
     */
    async saveToFile(filepath) {
        const data = {
            timestamp: new Date().toISOString(),
            totalMCPs: this.mcps.length,
            mcps: this.mcps
        };
        fs.writeFileSync(filepath, JSON.stringify(data, null, 2));
        console.log(`💾 Saved ${this.mcps.length} MCPs to ${filepath}`);
    }
}
// Run crawler if executed directly
// Check if this is the main module using import.meta.url
const isMainModule = import.meta.url === `file://${process.argv[1]}`;
if (isMainModule) {
    const crawler = new OfficialMCPCrawler();
    (async () => {
        await crawler.crawlOfficialRegistry();
        await crawler.crawlCommunityMCPs();
        await crawler.saveToFile('./data/crawled-mcps.json');
    })();
}