cf-memory-mcp
Version:
Best-in-class MCP server with CONTEXTUAL CHUNKING (Anthropic-style, 35-67% better retrieval), Optimized LLM stack (Llama-3.1-8B), BGE-M3 embeddings, Query Expansion Caching, Hybrid Embedding Strategy, and Unified Project Intelligence
597 lines (528 loc) • 21.9 kB
JavaScript
/**
* CF Memory MCP - Portable MCP Server
*
* A portable MCP (Model Context Protocol) server for AI memory storage
* using Cloudflare infrastructure. This executable connects to a deployed
* Cloudflare Worker to provide memory storage capabilities for AI agents.
*
* Usage: npx cf-memory-mcp
*
* @author John Lam <johnlam90@gmail.com>
* @license MIT
*/
const https = require('https');
const { URL } = require('url');
const os = require('os');
const process = require('process');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
// Configuration
const STREAMABLE_HTTP_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/mcp/message';
const LEGACY_SERVER_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/mcp/message';
const PACKAGE_VERSION = require('../package.json').version;
const TIMEOUT_MS = 60000; // Increased timeout for batch operations
const CONNECT_TIMEOUT_MS = 10000;
// Get API key from environment variable (will be checked later)
const API_KEY = process.env.CF_MEMORY_API_KEY;
/**
* Cross-platform MCP stdio bridge
* Handles communication between MCP clients and the Cloudflare Worker
*/
class CFMemoryMCP {
constructor() {
this.streamableHttpUrl = STREAMABLE_HTTP_URL;
this.legacyServerUrl = LEGACY_SERVER_URL;
this.userAgent = `cf-memory-mcp/${PACKAGE_VERSION} (${os.platform()} ${os.arch()}; Node.js ${process.version})`;
this.useStreamableHttp = true; // Try Streamable HTTP first
// Handle process termination gracefully
process.on('SIGINT', () => this.shutdown('SIGINT'));
process.on('SIGTERM', () => this.shutdown('SIGTERM'));
process.on('uncaughtException', (error) => {
this.logError('Uncaught exception:', error);
this.shutdown('ERROR');
});
// Set up stdio encoding
process.stdin.setEncoding('utf8');
process.stdout.setEncoding('utf8');
this.logDebug('CF Memory MCP server starting...');
this.logDebug(`Streamable HTTP URL: ${this.streamableHttpUrl}`);
this.logDebug(`Legacy Server URL: ${this.legacyServerUrl}`);
this.logDebug(`User Agent: ${this.userAgent}`);
}
/**
* Log debug messages to stderr (won't interfere with MCP communication)
*/
logDebug(message) {
if (process.env.DEBUG || process.env.MCP_DEBUG) {
process.stderr.write(`[DEBUG] ${new Date().toISOString()} ${message}\n`);
}
}
/**
* Log error messages to stderr
*/
logError(message, error = null) {
const timestamp = new Date().toISOString();
process.stderr.write(`[ERROR] ${timestamp} ${message}\n`);
if (error && error.stack) {
process.stderr.write(`[ERROR] ${timestamp} ${error.stack}\n`);
}
}
/**
* Start listening for MCP messages on stdin
*/
async start() {
try {
// Skip connectivity test in MCP mode - it will be tested when first request is made
this.logDebug('Starting MCP message processing...');
await this.processStdio();
} catch (error) {
this.logError('Failed to start MCP server:', error);
process.exit(1);
}
}
/**
* Test connectivity to the Cloudflare Worker
*/
async testConnectivity() {
this.logDebug('Testing connectivity to Cloudflare Worker...');
const testMessage = {
jsonrpc: '2.0',
id: 'connectivity-test',
method: 'initialize',
params: {
protocolVersion: '2025-03-26',
capabilities: { tools: {} },
clientInfo: {
name: 'cf-memory-mcp',
version: PACKAGE_VERSION
}
}
};
try {
// Try Streamable HTTP first
const response = await this.makeRequest(testMessage);
if (response.error) {
throw new Error(`Server responded with error: ${response.error.message}`);
}
this.logDebug('Connectivity test successful with Streamable HTTP');
} catch (error) {
// If Streamable HTTP fails, try legacy endpoint
this.logDebug('Streamable HTTP failed, trying legacy endpoint...');
this.useStreamableHttp = false;
try {
const response = await this.makeRequest(testMessage);
if (response.error) {
throw new Error(`Server responded with error: ${response.error.message}`);
}
this.logDebug('Connectivity test successful with legacy endpoint');
} catch (legacyError) {
throw new Error(`Cannot connect to CF Memory server: ${error.message} (Legacy also failed: ${legacyError.message})`);
}
}
}
/**
* Process stdio input/output for MCP communication
*/
async processStdio() {
let buffer = '';
// Handle stdin data
for await (const chunk of process.stdin) {
buffer += chunk;
// Process complete JSON-RPC messages (one per line)
let newlineIndex;
while ((newlineIndex = buffer.indexOf('\n')) !== -1) {
const line = buffer.slice(0, newlineIndex).trim();
buffer = buffer.slice(newlineIndex + 1);
if (line) {
await this.handleMessage(line);
}
}
}
// Process any remaining buffer content
if (buffer.trim()) {
await this.handleMessage(buffer.trim());
}
this.logDebug('Stdin closed, shutting down...');
}
/**
* Handle a single MCP message
*/
async handleMessage(messageStr) {
try {
const message = JSON.parse(messageStr);
this.logDebug(`Processing message: ${message.method} (id: ${message.id})`);
// Handle lifecycle methods locally
if (message.method === 'initialize') {
const response = {
jsonrpc: '2.0',
id: message.id,
result: {
protocolVersion: '2025-03-26',
capabilities: {
tools: {}
},
serverInfo: {
name: 'cf-memory-mcp',
version: PACKAGE_VERSION
}
}
};
process.stdout.write(JSON.stringify(response) + '\n');
return;
}
if (message.method === 'notifications/initialized') {
this.logDebug('MCP handshake completed');
return;
}
// Intercept index_project tool call to perform local scanning
if (message.method === 'tools/call' && message.params && message.params.name === 'index_project') {
await this.handleIndexProject(message);
return;
}
const response = await this.makeRequest(message);
// Send response to stdout
process.stdout.write(JSON.stringify(response) + '\n');
} catch (error) {
this.logError('Error handling message:', error);
// Send error response
const errorResponse = {
jsonrpc: '2.0',
id: null,
error: {
code: -32700,
message: 'Parse error',
data: error.message
}
};
process.stdout.write(JSON.stringify(errorResponse) + '\n');
}
}
/**
* Handle local project indexing - scans local files and sends them via MCP
*/
async handleIndexProject(message) {
const { project_path, project_name, include_patterns, exclude_patterns } = message.params.arguments;
const resolvedPath = path.resolve(project_path);
const name = project_name || path.basename(resolvedPath);
this.logDebug(`Intercepted index_project for: ${resolvedPath} (${name})`);
try {
// 1. Scan Local Files
this.logDebug(`Scanning files in ${resolvedPath}...`);
const files = this.scanDirectory(resolvedPath, '', include_patterns, exclude_patterns);
this.logDebug(`Found ${files.length} files to index.`);
if (files.length === 0) {
const response = {
jsonrpc: '2.0',
id: message.id,
result: {
content: [{
type: 'text',
text: JSON.stringify({
project_id: null,
project_name: name,
files_found: 0,
files_indexed: 0,
chunks_created: 0,
status: 'complete',
message: 'No matching files found in directory'
}, null, 2)
}]
}
};
process.stdout.write(JSON.stringify(response) + '\n');
return;
}
// 2. Send files directly via MCP tool call (batched to avoid timeouts)
const BATCH_SIZE = 20; // Process 20 files at a time
let totalIndexed = 0;
let totalChunks = 0;
let projectId = null;
for (let i = 0; i < files.length; i += BATCH_SIZE) {
const batch = files.slice(i, i + BATCH_SIZE);
this.logDebug(`Processing batch ${Math.floor(i/BATCH_SIZE) + 1}/${Math.ceil(files.length/BATCH_SIZE)} (${batch.length} files)`);
const projectResult = await this.makeRequest({
jsonrpc: '2.0',
id: `index-batch-${Date.now()}-${i}`,
method: 'tools/call',
params: {
name: 'index_project',
arguments: {
project_path: resolvedPath,
project_name: name,
files: batch.map(f => ({
path: f.relativePath,
content: f.content
})),
include_patterns,
exclude_patterns
}
}
});
if (projectResult.error) {
this.logError(`Batch ${i} failed:`, projectResult.error);
continue;
}
try {
const content = JSON.parse(projectResult.result.content[0].text);
projectId = content.project_id;
totalIndexed += content.files_indexed || 0;
totalChunks += content.chunks_created || 0;
} catch (e) {
this.logError('Could not parse batch result:', e);
}
}
// 3. Return aggregated success
const response = {
jsonrpc: '2.0',
id: message.id,
result: {
content: [{
type: 'text',
text: JSON.stringify({
project_id: projectId,
project_name: name,
files_found: files.length,
files_indexed: totalIndexed,
chunks_created: totalChunks,
status: 'complete'
}, null, 2)
}]
}
};
process.stdout.write(JSON.stringify(response) + '\n');
} catch (error) {
this.logError('Index project failed:', error);
const response = {
jsonrpc: '2.0',
id: message.id,
error: {
code: -32000,
message: `Indexing failed: ${error.message}`
}
};
process.stdout.write(JSON.stringify(response) + '\n');
}
}
/**
* Recursive directory scan with filtering
*/
scanDirectory(dir, basePath = '', includePatterns = null, excludePatterns = null) {
let files = [];
// Default exclusions (always excluded)
const DEFAULT_EXCLUDE_DIRS = [
// Version control
'.git', '.svn', '.hg',
// Dependencies
'node_modules', 'vendor', 'packages',
// Build outputs
'dist', 'build', 'out', '.next', '.nuxt',
// Python
'__pycache__', '.pytest_cache', 'venv', '.venv',
// Test coverage
'coverage', '.coverage', '.nyc_output',
// IDE/Editor history & settings
'.history', '.vscode', '.idea', '.vs',
// Cloudflare/tooling
'.wrangler', '.turbo', '.cache'
];
// Default file extensions to include
const DEFAULT_INCLUDE_EXTS = [
'.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
'.py', '.rb', '.go', '.rs', '.java', '.kt', '.scala',
'.cs', '.cpp', '.c', '.h', '.hpp', '.swift', '.php',
'.sql', '.sh', '.bash',
'.json', '.yaml', '.yml', '.toml',
'.html', '.css', '.scss', '.vue', '.svelte',
'.md', '.mdx'
];
// Merge custom exclude patterns
const effectiveExcludes = excludePatterns
? [...DEFAULT_EXCLUDE_DIRS, ...excludePatterns]
: DEFAULT_EXCLUDE_DIRS;
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
const relPath = path.join(basePath, entry.name);
if (entry.isDirectory()) {
// Check if directory should be excluded
const shouldExclude = effectiveExcludes.some(pattern => {
if (pattern.includes('*')) {
// Simple glob matching
const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
return regex.test(entry.name);
}
return entry.name === pattern;
});
if (!shouldExclude) {
files = files.concat(this.scanDirectory(fullPath, relPath, includePatterns, excludePatterns));
}
} else if (entry.isFile()) {
const ext = path.extname(entry.name).toLowerCase();
// Check if file should be excluded
const fileExcluded = effectiveExcludes.some(pattern => {
if (pattern.startsWith('*.')) {
return ext === pattern.substring(1);
}
return entry.name === pattern || relPath.includes(pattern);
});
if (fileExcluded) continue;
// Check if file matches include patterns
let shouldInclude = false;
if (includePatterns && includePatterns.length > 0) {
shouldInclude = includePatterns.some(pattern => {
if (pattern.startsWith('*.')) {
return ext === pattern.substring(1);
}
return entry.name.includes(pattern) || relPath.includes(pattern);
});
} else {
// Use default extensions
shouldInclude = DEFAULT_INCLUDE_EXTS.includes(ext);
}
if (shouldInclude) {
try {
const content = fs.readFileSync(fullPath, 'utf8');
// Skip huge files > 500KB
if (content.length < 512 * 1024) {
files.push({
relativePath: relPath,
content: content
});
} else {
this.logDebug(`Skipping large file (${Math.round(content.length/1024)}KB): ${relPath}`);
}
} catch (e) {
this.logDebug(`Could not read file: ${relPath}`);
}
}
}
}
} catch (e) {
this.logError(`Scan failed for ${dir}:`, e);
}
return files;
}
/**
* Make HTTP request to the Cloudflare Worker (MCP JSON-RPC)
*/
async makeRequest(message) {
return new Promise((resolve) => {
const serverUrl = this.useStreamableHttp ? this.streamableHttpUrl : this.legacyServerUrl;
const url = new URL(serverUrl);
const postData = JSON.stringify(message);
const options = {
hostname: url.hostname,
port: url.port || 443,
path: url.pathname,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Accept-Encoding': 'identity',
'User-Agent': this.userAgent,
'Authorization': `Bearer ${API_KEY}`,
'X-API-Key': API_KEY, // Also include for backwards compatibility
'Content-Length': Buffer.byteLength(postData)
},
timeout: TIMEOUT_MS
};
const req = https.request(options, (res) => {
let body = '';
res.on('data', (chunk) => {
body += chunk;
});
res.on('end', () => {
try {
const response = JSON.parse(body);
resolve(response);
} catch (error) {
resolve({
jsonrpc: '2.0',
id: message.id || null,
error: {
code: -32603,
message: 'Invalid JSON response from server',
data: error.message
}
});
}
});
});
req.on('error', (error) => {
resolve({
jsonrpc: '2.0',
id: message.id || null,
error: {
code: -32603,
message: 'Network error',
data: `Failed to connect to ${serverUrl}: ${error.message}`
}
});
});
req.on('timeout', () => {
req.destroy();
resolve({
jsonrpc: '2.0',
id: message.id || null,
error: {
code: -32603,
message: 'Request timeout',
data: `Server did not respond within ${TIMEOUT_MS}ms`
}
});
});
req.write(postData);
req.end();
});
}
/**
* Graceful shutdown
*/
shutdown(reason = 'UNKNOWN') {
this.logDebug(`Shutting down (reason: ${reason})`);
process.exit(0);
}
}
// Handle command line arguments
if (process.argv.includes('--version') || process.argv.includes('-v')) {
console.log(`cf-memory-mcp v${PACKAGE_VERSION}`);
process.exit(0);
}
if (process.argv.includes('--help') || process.argv.includes('-h')) {
console.log(`
CF Memory MCP v${PACKAGE_VERSION}
A portable MCP (Model Context Protocol) server for AI memory storage using Cloudflare infrastructure.
Usage:
npx cf-memory-mcp Start the MCP server
npx cf-memory-mcp --version Show version
npx cf-memory-mcp --help Show this help
Environment Variables:
CF_MEMORY_API_KEY=<key> Your CF Memory API key (required)
DEBUG=1 Enable debug logging
MCP_DEBUG=1 Enable MCP debug logging
For more information, visit: https://github.com/johnlam90/cf-memory-mcp
`);
process.exit(0);
}
// Check API key before starting server
if (!API_KEY) {
console.error('Error: CF_MEMORY_API_KEY environment variable is required');
console.error('');
console.error('Please set your API key:');
console.error(' export CF_MEMORY_API_KEY="your-api-key-here"');
console.error('');
console.error('Or run with:');
console.error(' CF_MEMORY_API_KEY="your-api-key-here" npx cf-memory-mcp');
console.error('');
console.error('Get your API key from: https://cf-memory-mcp.johnlam90.workers.dev');
process.exit(1);
}
// Start the MCP server
const server = new CFMemoryMCP();
server.start().catch(error => {
console.error('Failed to start CF Memory MCP server:', error.message);
process.exit(1);
});