alphe-redis-mcp-server/src/core/redis-client.ts

The most comprehensive Redis MCP Server for Alphe.AI - Optimized for sub-5 second response times with multi-layer caching

import Redis from 'ioredis';
import { Redis as UpstashRedis } from '@upstash/redis';
import LRUCache from 'lru-cache';
import { createClient as createSupabaseClient, SupabaseClient } from '@supabase/supabase-js';
import { MilvusClient } from '@zilliz/milvus2-sdk-node';
import { 
  RedisConfig, 
  UpstashConfig, 
  ZillizConfig, 
  SupabaseConfig,
  PerformanceConfig,
  CacheOptions,
  CacheEntry,
  CacheStats,
  PerformanceMetrics,
  RedisResponse,
  HealthCheck,
  ConnectionPool,
  MemoryItem,
  SearchOptions,
  SearchResult
} from '../types/index.js';

export class AlpheRedisClient {
  private redisClient: Redis | null = null;
  private upstashClient: UpstashRedis | null = null;
  private supabaseClient: SupabaseClient | null = null;
  private milvusClient: MilvusClient | null = null;
  
  // Multi-layer caching
  private memoryCache: LRUCache<string, CacheEntry>;
  private compressionEnabled: boolean = true;
  private performanceMetrics: PerformanceMetrics;
  
  // Configuration
  private config: {
    redis?: RedisConfig;
    upstash?: UpstashConfig;
    zilliz?: ZillizConfig;
    supabase?: SupabaseConfig;
    performance: PerformanceConfig;
  };

  constructor(config: any) {
    this.config = config;
    
    // Initialize in-memory L1 cache
    this.memoryCache = new LRUCache<string, CacheEntry>({
      max: 10000, // Maximum number of items
      maxSize: config.performance.maxCacheSizeMB * 1024 * 1024, // Convert MB to bytes
      ttl: config.performance.cacheTtlSeconds * 1000, // Convert to milliseconds
      sizeCalculation: (value: CacheEntry) => value.size,
      dispose: (value: CacheEntry, key: string) => {
        console.log(`🗑️ Evicting cache entry: ${key}`);
      }
    });

    this.compressionEnabled = config.performance.enableCompression;
    
    // Initialize performance metrics
    this.performanceMetrics = {
      latency: { min: 0, max: 0, avg: 0, p50: 0, p95: 0, p99: 0 },
      throughput: { requestsPerSecond: 0, bytesPerSecond: 0 },
      memory: { used: 0, available: 0, fragmentation: 0 },
      connections: { active: 0, idle: 0, total: 0 },
      cache: {
        totalKeys: 0,
        totalHits: 0,
        totalMisses: 0,
        hitRate: 0,
        memoryUsage: 0,
        compressionRatio: 0,
        avgLatency: 0,
        topKeys: []
      }
    };
  }

  async initialize(): Promise<void> {
    console.log('🚀 Initializing Alphe Redis Client with multi-layer caching...');
    
    try {
      // Initialize Redis client
      if (this.config.redis) {
        await this.initializeRedis();
      }
      
      // Initialize Upstash client
      if (this.config.upstash) {
        await this.initializeUpstash();
      }
      
      // Initialize Zilliz client
      if (this.config.zilliz) {
        await this.initializeZilliz();
      }
      
      // Initialize Supabase client
      if (this.config.supabase) {
        await this.initializeSupabase();
      }
      
      console.log('✅ All clients initialized successfully');
    } catch (error) {
      console.error('❌ Failed to initialize clients:', error);
      throw error;
    }
  }

  private async initializeRedis(): Promise<void> {
    const config = this.config.redis!;
    
    if (config.cluster && config.nodes) {
      // Redis Cluster
      this.redisClient = new Redis.Cluster(config.nodes, {
        enableReadyCheck: config.enableReadyCheck ?? false,
        redisOptions: {
          password: config.password,
          username: config.username,
          connectTimeout: config.connectTimeout ?? 10000,
          commandTimeout: config.commandTimeout ?? 5000,
          retryDelayOnFailover: config.retryDelayOnFailover ?? 100,
          maxRetriesPerRequest: config.maxRetriesPerRequest ?? 3,
          lazyConnect: true
        }
      });
    } else {
      // Single Redis instance
      this.redisClient = new Redis({
        host: config.host ?? 'localhost',
        port: config.port ?? 6379,
        password: config.password,
        username: config.username,
        db: config.db ?? 0,
        connectTimeout: config.connectTimeout ?? 10000,
        commandTimeout: config.commandTimeout ?? 5000,
        retryDelayOnFailover: config.retryDelayOnFailover ?? 100,
        maxRetriesPerRequest: config.maxRetriesPerRequest ?? 3,
        enableReadyCheck: config.enableReadyCheck ?? false,
        lazyConnect: true,
        keepAlive: config.keepAlive ?? 30000,
        family: config.family ?? 4
      });
    }

    // Event handlers
    this.redisClient.on('connect', () => {
      console.log('✅ Redis connected');
      this.performanceMetrics.connections.active++;
    });

    this.redisClient.on('error', (error) => {
      console.error('❌ Redis error:', error);
    });

    this.redisClient.on('ready', () => {
      console.log('🚀 Redis ready');
    });

    this.redisClient.on('close', () => {
      console.log('👋 Redis disconnected');
      if (this.performanceMetrics.connections.active > 0) {
        this.performanceMetrics.connections.active--;
      }
    });

    await this.redisClient.connect();
  }

  private async initializeUpstash(): Promise<void> {
    const config = this.config.upstash!;
    
    this.upstashClient = new UpstashRedis({
      url: config.url,
      token: config.token,
      retry: {
        retries: 3,
        backoff: (retryCount: number) => Math.exp(retryCount) * 50,
      },
      enableTelemetry: false
    });
    
    // Test connection
    await this.upstashClient.ping();
    console.log('✅ Upstash Redis connected');
  }

  private async initializeZilliz(): Promise<void> {
    const config = this.config.zilliz!;
    
    this.milvusClient = new MilvusClient({
      address: config.endpoint,
      token: config.token,
      username: config.username,
      password: config.password,
      ssl: true,
      timeout: 10000
    });
    
    console.log('✅ Zilliz/Milvus connected');
  }

  private async initializeSupabase(): Promise<void> {
    const config = this.config.supabase!;
    
    this.supabaseClient = createSupabaseClient(
      config.url,
      config.serviceRoleKey,
      {
        auth: {
          persistSession: false,
          autoRefreshToken: false
        },
        global: {
          headers: {
            'x-client-info': 'alphe-redis-mcp-server'
          }
        }
      }
    );
    
    // Test connection
    const { error } = await this.supabaseClient.from('mcp_cache').select('count').limit(1);
    if (error && !error.message.includes('relation "mcp_cache" does not exist')) {
      throw new Error(`Supabase connection failed: ${error.message}`);
    }
    
    console.log('✅ Supabase connected');
  }

  // Multi-layer GET with automatic fallback
  async get<T = string>(key: string, options?: CacheOptions): Promise<RedisResponse<T>> {
    const startTime = performance.now();
    
    try {
      // L1: Memory cache
      const memoryResult = this.getFromMemoryCache<T>(key);
      if (memoryResult.data !== undefined) {
        this.performanceMetrics.cache.totalHits++;
        return {
          success: true,
          data: memoryResult.data,
          cached: true,
          latency: performance.now() - startTime,
          fromLevel: 'memory'
        };
      }

      // L2: Redis cache
      if (this.redisClient) {
        const redisValue = await this.redisClient.get(key);
        if (redisValue !== null) {
          const data = this.deserializeValue<T>(redisValue);
          // Store in memory cache
          this.setInMemoryCache(key, data, options);
          this.performanceMetrics.cache.totalHits++;
          return {
            success: true,
            data,
            cached: true,
            latency: performance.now() - startTime,
            fromLevel: 'redis'
          };
        }
      }

      // L3: Upstash fallback
      if (this.upstashClient) {
        const upstashValue = await this.upstashClient.get(key);
        if (upstashValue !== null) {
          const data = typeof upstashValue === 'string' 
            ? this.deserializeValue<T>(upstashValue)
            : upstashValue as T;
          
          // Store in both Redis and memory cache
          if (this.redisClient) {
            await this.redisClient.set(key, this.serializeValue(data), 'EX', options?.ttl ?? this.config.performance.cacheTtlSeconds);
          }
          this.setInMemoryCache(key, data, options);
          this.performanceMetrics.cache.totalHits++;
          return {
            success: true,
            data,
            cached: true,
            latency: performance.now() - startTime,
            fromLevel: 'redis'
          };
        }
      }

      // L4: Check Zilliz for vector-based retrieval (if key suggests it's a search query)
      if (this.milvusClient && key.startsWith('search:')) {
        const searchResult = await this.searchInZilliz(key);
        if (searchResult.length > 0) {
          const data = searchResult as unknown as T;
          // Cache the result
          await this.set(key, data, { ttl: options?.ttl ?? 1800 }); // 30 min cache
          return {
            success: true,
            data,
            cached: false,
            latency: performance.now() - startTime,
            fromLevel: 'zilliz'
          };
        }
      }

      // L5: Check Supabase for persistent storage
      if (this.supabaseClient && key.startsWith('persistent:')) {
        const supabaseResult = await this.getFromSupabase(key);
        if (supabaseResult !== null) {
          const data = supabaseResult as T;
          // Cache the result in all layers
          await this.set(key, data, { ttl: options?.ttl ?? 3600 }); // 1 hour cache
          return {
            success: true,
            data,
            cached: false,
            latency: performance.now() - startTime,
            fromLevel: 'supabase'
          };
        }
      }

      this.performanceMetrics.cache.totalMisses++;
      return {
        success: false,
        error: 'Key not found in any cache layer',
        latency: performance.now() - startTime
      };

    } catch (error) {
      this.performanceMetrics.cache.totalMisses++;
      return {
        success: false,
        error: error instanceof Error ? error.message : 'Unknown error',
        latency: performance.now() - startTime
      };
    }
  }

  // Multi-layer SET with intelligent caching
  async set<T = string>(key: string, value: T, options?: CacheOptions): Promise<RedisResponse<boolean>> {
    const startTime = performance.now();
    
    try {
      const serializedValue = this.serializeValue(value);
      const ttl = options?.ttl ?? this.config.performance.cacheTtlSeconds;
      
      // Store in all available layers based on options
      const promises: Promise<any>[] = [];
      
      // L1: Memory cache (always store)
      this.setInMemoryCache(key, value, options);
      
      // L2: Redis
      if (this.redisClient) {
        promises.push(this.redisClient.set(key, serializedValue, 'EX', ttl));
      }
      
      // L3: Upstash (for backup/distribution)
      if (this.upstashClient) {
        promises.push(this.upstashClient.set(key, serializedValue, { ex: ttl }));
      }
      
      // L4: Zilliz (for vector data)
      if (this.milvusClient && this.isVectorData(value)) {
        promises.push(this.storeInZilliz(key, value as any));
      }
      
      // L5: Supabase (for persistent data)
      if (this.supabaseClient && options?.namespace === 'persistent') {
        promises.push(this.storeInSupabase(key, value));
      }
      
      await Promise.all(promises);
      
      return {
        success: true,
        data: true,
        latency: performance.now() - startTime
      };
      
    } catch (error) {
      return {
        success: false,
        data: false,
        error: error instanceof Error ? error.message : 'Unknown error',
        latency: performance.now() - startTime
      };
    }
  }

  // Batch operations for better performance
  async mget(keys: string[]): Promise<RedisResponse<(string | null)[]>> {
    const startTime = performance.now();
    
    try {
      const results: (string | null)[] = new Array(keys.length);
      const missingKeys: { index: number; key: string }[] = [];
      
      // Check memory cache first
      for (let i = 0; i < keys.length; i++) {
        const memoryResult = this.getFromMemoryCache(keys[i]);
        if (memoryResult.data !== undefined) {
          results[i] = this.serializeValue(memoryResult.data);
          this.performanceMetrics.cache.totalHits++;
        } else {
          missingKeys.push({ index: i, key: keys[i] });
        }
      }
      
      // Fetch missing keys from Redis
      if (missingKeys.length > 0 && this.redisClient) {
        const redisKeys = missingKeys.map(mk => mk.key);
        const redisResults = await this.redisClient.mget(...redisKeys);
        
        for (let i = 0; i < redisResults.length; i++) {
          const value = redisResults[i];
          const { index, key } = missingKeys[i];
          
          if (value !== null) {
            results[index] = value;
            // Cache in memory
            this.setInMemoryCache(key, this.deserializeValue(value));
            this.performanceMetrics.cache.totalHits++;
          } else {
            results[index] = null;
            this.performanceMetrics.cache.totalMisses++;
          }
        }
      }
      
      return {
        success: true,
        data: results,
        latency: performance.now() - startTime
      };
      
    } catch (error) {
      return {
        success: false,
        error: error instanceof Error ? error.message : 'Unknown error',
        latency: performance.now() - startTime
      };
    }
  }

  // Advanced search with semantic similarity
  async semanticSearch(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
    if (!this.milvusClient) {
      throw new Error('Zilliz/Milvus client not initialized for semantic search');
    }

    const cacheKey = `semantic_search:${query}:${JSON.stringify(options)}`;
    
    // Check cache first
    const cachedResult = await this.get<SearchResult[]>(cacheKey);
    if (cachedResult.success && cachedResult.data) {
      return cachedResult.data;
    }

    // Perform semantic search in Zilliz
    const results = await this.searchInZilliz(query, options);
    
    // Cache results for future queries
    await this.set(cacheKey, results, { ttl: 900, namespace: 'search' }); // 15 min cache
    
    return results;
  }

  // Performance monitoring
  async getPerformanceMetrics(): Promise<PerformanceMetrics> {
    const cacheSize = this.memoryCache.size;
    const cacheCalculatedSize = this.memoryCache.calculatedSize ?? 0;
    
    this.performanceMetrics.cache.totalKeys = cacheSize;
    this.performanceMetrics.cache.memoryUsage = cacheCalculatedSize;
    this.performanceMetrics.cache.hitRate = 
      this.performanceMetrics.cache.totalHits / 
      (this.performanceMetrics.cache.totalHits + this.performanceMetrics.cache.totalMisses) || 0;

    if (this.redisClient) {
      try {
        const info = await this.redisClient.info('memory');
        const memoryMatch = info.match(/used_memory:(\d+)/);
        if (memoryMatch) {
          this.performanceMetrics.memory.used = parseInt(memoryMatch[1]);
        }
      } catch (error) {
        console.warn('Could not fetch Redis memory info:', error);
      }
    }

    return { ...this.performanceMetrics };
  }

  // Health check for all systems
  async healthCheck(): Promise<HealthCheck> {
    const health: HealthCheck = {
      redis: { connected: false, latency: 0, memory: 0, version: 'unknown' },
      zilliz: { connected: false, latency: 0, collections: 0 },
      supabase: { connected: false, latency: 0 },
      overall: 'unhealthy'
    };

    try {
      // Redis health check
      if (this.redisClient) {
        const startTime = performance.now();
        const pong = await this.redisClient.ping();
        health.redis.latency = performance.now() - startTime;
        health.redis.connected = pong === 'PONG';
        
        const info = await this.redisClient.info();
        const versionMatch = info.match(/redis_version:([^\r\n]+)/);
        if (versionMatch) health.redis.version = versionMatch[1];
      }

      // Zilliz health check
      if (this.milvusClient) {
        const startTime = performance.now();
        const collections = await this.milvusClient.listCollections();
        health.zilliz.latency = performance.now() - startTime;
        health.zilliz.connected = true;
        health.zilliz.collections = collections.collection_names?.length ?? 0;
      }

      // Supabase health check
      if (this.supabaseClient) {
        const startTime = performance.now();
        const { error } = await this.supabaseClient.from('mcp_cache').select('count').limit(1);
        health.supabase.latency = performance.now() - startTime;
        health.supabase.connected = !error || error.message.includes('relation "mcp_cache" does not exist');
      }

      // Determine overall health
      const connectedSystems = [
        health.redis.connected,
        health.zilliz.connected,
        health.supabase.connected
      ].filter(Boolean).length;

      if (connectedSystems >= 2) {
        health.overall = 'healthy';
      } else if (connectedSystems >= 1) {
        health.overall = 'degraded';
      }

    } catch (error) {
      console.error('Health check error:', error);
    }

    return health;
  }

  // Utility methods
  private getFromMemoryCache<T>(key: string): { data: T | undefined; cached: boolean } {
    const entry = this.memoryCache.get(key);
    if (entry && Date.now() - entry.timestamp < entry.ttl * 1000) {
      entry.hits++;
      return { data: entry.data as T, cached: true };
    }
    return { data: undefined, cached: false };
  }

  private setInMemoryCache<T>(key: string, value: T, options?: CacheOptions): void {
    const serialized = this.serializeValue(value);
    const compressed = this.compressionEnabled && serialized.length > 1000;
    const size = compressed ? Math.floor(serialized.length / 2) : serialized.length;
    
    const entry: CacheEntry<T> = {
      data: value,
      timestamp: Date.now(),
      ttl: options?.ttl ?? this.config.performance.cacheTtlSeconds,
      hits: 1,
      size,
      compressed,
      tags: options?.tags ?? [],
      priority: options?.priority ?? 1
    };
    
    this.memoryCache.set(key, entry);
  }

  private serializeValue<T>(value: T): string {
    if (typeof value === 'string') return value;
    return JSON.stringify(value);
  }

  private deserializeValue<T>(value: string): T {
    try {
      return JSON.parse(value) as T;
    } catch {
      return value as unknown as T;
    }
  }

  private isVectorData(value: any): boolean {
    return Array.isArray(value) && value.length > 0 && typeof value[0] === 'number';
  }

  private async searchInZilliz(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
    // This would implement actual semantic search using embeddings
    // For now, return empty results
    return [];
  }

  private async storeInZilliz(key: string, value: any): Promise<void> {
    // Implement vector storage in Zilliz
    console.log(`Storing vector data in Zilliz: ${key}`);
  }

  private async getFromSupabase(key: string): Promise<any> {
    if (!this.supabaseClient) return null;
    
    try {
      const { data, error } = await this.supabaseClient
        .from('mcp_cache')
        .select('cached_response')
        .eq('cache_key', key)
        .single();
        
      if (error) return null;
      return data?.cached_response;
    } catch {
      return null;
    }
  }

  private async storeInSupabase(key: string, value: any): Promise<void> {
    if (!this.supabaseClient) return;
    
    try {
      await this.supabaseClient
        .from('mcp_cache')
        .upsert({
          cache_key: key,
          cached_response: value,
          created_at: new Date().toISOString(),
          expires_at: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString() // 24 hours
        });
    } catch (error) {
      console.error('Failed to store in Supabase:', error);
    }
  }

  async disconnect(): Promise<void> {
    console.log('🔌 Disconnecting all clients...');
    
    const disconnectPromises: Promise<void>[] = [];
    
    if (this.redisClient) {
      disconnectPromises.push(this.redisClient.quit());
    }
    
    if (this.milvusClient) {
      // Milvus client doesn't have explicit disconnect
    }
    
    await Promise.all(disconnectPromises);
    
    this.memoryCache.clear();
    
    console.log('👋 All clients disconnected');
  }
}