UNPKG

personal-data-wallet-sdk

Version:

TypeScript SDK for Personal Data Wallet - Decentralized memory system with AI embeddings, HNSW vector search, SEAL encryption and Walrus storage

825 lines (720 loc) โ€ข 28.1 kB
/** * MemoryDecryptionPipeline - Seamless SEAL-based Memory Decryption * * Provides comprehensive decryption capabilities for encrypted memories using * Mysten's SEAL SDK with official testnet key servers and configurable infrastructure. * * Features: * - ๐Ÿ” Seamless SEAL decryption integration * - ๐Ÿ”‘ Automatic key server configuration (testnet/mainnet) * - โšก Batch decryption optimization * - ๐Ÿ›ก๏ธ Secure session key management * - ๐Ÿ”„ Automatic retry and fallback mechanisms * - ๐Ÿ“Š Decryption analytics and monitoring * - ๐ŸŒ Environment-based configuration */ import { SealClient, SessionKey } from '@mysten/seal'; import { Transaction } from '@mysten/sui/transactions'; import { fromHex, toHex } from '@mysten/sui/utils'; import { EncryptionService } from '../services/EncryptionService'; import { StorageManager } from '../infrastructure/walrus/StorageManager'; import { UnifiedMemoryResult } from '../retrieval/MemoryRetrievalService'; // Key server configurations from SEAL documentation export interface KeyServerConfig { name: string; mode: 'open' | 'permissioned'; objectId: string; url: string; provider: string; network: 'testnet' | 'mainnet'; weight?: number; isDefault?: boolean; } export interface DecryptionConfig { // Key server configuration keyServers?: KeyServerConfig[]; defaultKeyServerMode?: 'open' | 'permissioned'; customKeyServerUrl?: string; customKeyServerObjectId?: string; // Session management sessionKeyTTL?: number; // minutes maxSessionKeys?: number; autoRefreshSession?: boolean; // Performance options enableBatchDecryption?: boolean; batchSize?: number; maxConcurrentDecryptions?: number; decryptionTimeout?: number; // ms // Fallback options enableFallback?: boolean; maxRetryAttempts?: number; retryDelayMs?: number; // Security options verifyKeyServers?: boolean; enableDecryptionAudit?: boolean; requireOwnershipVerification?: boolean; } export interface DecryptionRequest { memoryId: string; encryptedContent: string; contentHash?: string; userAddress: string; ownerAddress?: string; sessionKey?: SessionKey; metadata?: Record<string, any>; } export interface DecryptionResult { memoryId: string; decryptedContent: string; contentHash: string; isVerified: boolean; decryptionTime: number; keyServerUsed: string; sessionKeyId: string; } export interface BatchDecryptionResult { successful: DecryptionResult[]; failed: Array<{ memoryId: string; error: string; retryCount: number; }>; stats: { totalRequests: number; successCount: number; failureCount: number; totalProcessingTime: number; averageDecryptionTime: number; keyServerPerformance: Record<string, { requests: number; successes: number; averageTime: number; }>; }; } /** * Memory Decryption Pipeline Service */ export class MemoryDecryptionPipeline { private sealClient: SealClient | null = null; private encryptionService: EncryptionService; private storageManager: StorageManager; private config: DecryptionConfig; // Session key management private sessionKeys = new Map<string, SessionKey>(); private sessionKeyTimestamps = new Map<string, number>(); // Decryption cache private decryptionCache = new Map<string, { content: string; timestamp: number }>(); private readonly CACHE_TTL = 30 * 60 * 1000; // 30 minutes // Performance monitoring private decryptionStats = { totalDecryptions: 0, successfulDecryptions: 0, failedDecryptions: 0, totalDecryptionTime: 0, keyServerStats: new Map<string, { requests: number; successes: number; totalTime: number }>() }; // Official Mysten Labs testnet key servers from documentation private static readonly DEFAULT_TESTNET_SERVERS: KeyServerConfig[] = [ { name: 'mysten-testnet-1', mode: 'open', objectId: '0x73d05d62c18d9374e3ea529e8e0ed6161da1a141a94d3f76ae3fe4e99356db75', url: 'https://seal-key-server-testnet-1.mystenlabs.com', provider: 'Mysten Labs', network: 'testnet', weight: 1, isDefault: true }, { name: 'mysten-testnet-2', mode: 'open', objectId: '0xf5d14a81a982144ae441cd7d64b09027f116a468bd36e7eca494f750591623c8', url: 'https://seal-key-server-testnet-2.mystenlabs.com', provider: 'Mysten Labs', network: 'testnet', weight: 1, isDefault: true } ]; // Additional verified testnet key servers private static readonly VERIFIED_TESTNET_SERVERS: KeyServerConfig[] = [ { name: 'ruby-nodes-testnet', mode: 'open', objectId: '0x6068c0acb197dddbacd4746a9de7f025b2ed5a5b6c1b1ab44dade4426d141da2', url: 'https://seal-testnet.api.rubynodes.io', provider: 'Ruby Nodes', network: 'testnet', weight: 1 }, { name: 'nodeinfra-testnet', mode: 'open', objectId: '0x5466b7df5c15b508678d51496ada8afab0d6f70a01c10613123382b1b8131007', url: 'https://open-seal-testnet.nodeinfra.com', provider: 'NodeInfra', network: 'testnet', weight: 1 }, { name: 'studio-mirai-testnet', mode: 'open', objectId: '0x164ac3d2b3b8694b8181c13f671950004765c23f270321a45fdd04d40cccf0f2', url: 'https://open.key-server-testnet.seal.mirai.cloud', provider: 'Studio Mirai', network: 'testnet', weight: 1 }, { name: 'overclock-testnet', mode: 'open', objectId: '0x9c949e53c36ab7a9c484ed9e8b43267a77d4b8d70e79aa6b39042e3d4c434105', url: 'https://seal-testnet-open.overclock.run', provider: 'Overclock', network: 'testnet', weight: 1 }, { name: 'h2o-nodes-testnet', mode: 'open', objectId: '0x39cef09b24b667bc6ed54f7159d82352fe2d5dd97ca9a5beaa1d21aa774f25a2', url: 'https://seal-open.sui-testnet.h2o-nodes.com', provider: 'H2O Nodes', network: 'testnet', weight: 1 }, { name: 'triton-one-testnet', mode: 'open', objectId: '0x4cded1abeb52a22b6becb42a91d3686a4c901cf52eee16234214d0b5b2da4c46', url: 'https://seal.testnet.sui.rpcpool.com', provider: 'Triton One', network: 'testnet', weight: 1 } ]; constructor( encryptionService: EncryptionService, storageManager: StorageManager, config?: Partial<DecryptionConfig> ) { this.encryptionService = encryptionService; this.storageManager = storageManager; this.config = this.mergeWithDefaults(config || {}); this.initializeDecryptionPipeline(); } // ==================== INITIALIZATION ==================== /** * Merge user config with environment variables and defaults */ private mergeWithDefaults(userConfig: Partial<DecryptionConfig>): DecryptionConfig { // Load from environment variables const envConfig: Partial<DecryptionConfig> = { customKeyServerUrl: process.env.SEAL_KEY_SERVER_URL, customKeyServerObjectId: process.env.SEAL_KEY_SERVER_OBJECT_ID, sessionKeyTTL: process.env.SEAL_SESSION_TTL ? parseInt(process.env.SEAL_SESSION_TTL) : undefined, enableBatchDecryption: process.env.SEAL_ENABLE_BATCH === 'true', batchSize: process.env.SEAL_BATCH_SIZE ? parseInt(process.env.SEAL_BATCH_SIZE) : undefined, decryptionTimeout: process.env.SEAL_DECRYPTION_TIMEOUT ? parseInt(process.env.SEAL_DECRYPTION_TIMEOUT) : undefined, verifyKeyServers: process.env.SEAL_VERIFY_SERVERS !== 'false', // Default true enableDecryptionAudit: process.env.SEAL_ENABLE_AUDIT === 'true' }; // Default configuration const defaults: DecryptionConfig = { keyServers: this.getDefaultKeyServers(), defaultKeyServerMode: 'open', sessionKeyTTL: 60, // 1 hour maxSessionKeys: 100, autoRefreshSession: true, enableBatchDecryption: true, batchSize: 10, maxConcurrentDecryptions: 5, decryptionTimeout: 30000, // 30 seconds enableFallback: true, maxRetryAttempts: 3, retryDelayMs: 1000, verifyKeyServers: process.env.NODE_ENV === 'production', enableDecryptionAudit: false, requireOwnershipVerification: true }; return { ...defaults, ...envConfig, ...userConfig }; } /** * Get default key servers based on environment */ private getDefaultKeyServers(): KeyServerConfig[] { const network = process.env.SUI_NETWORK || 'testnet'; if (network === 'testnet') { // Use official Mysten Labs servers as primary, with fallbacks return [ ...MemoryDecryptionPipeline.DEFAULT_TESTNET_SERVERS, ...MemoryDecryptionPipeline.VERIFIED_TESTNET_SERVERS.slice(0, 2) // Add 2 fallback servers ]; } // For mainnet, would need to configure based on chosen providers return MemoryDecryptionPipeline.DEFAULT_TESTNET_SERVERS; } /** * Initialize the decryption pipeline */ private async initializeDecryptionPipeline(): Promise<void> { try { console.log('๐Ÿ”‘ Initializing SEAL Memory Decryption Pipeline...'); // Determine key servers to use let keyServers = this.config.keyServers || this.getDefaultKeyServers(); // Add custom key server if configured if (this.config.customKeyServerUrl && this.config.customKeyServerObjectId) { keyServers = [{ name: 'custom-server', mode: this.config.defaultKeyServerMode || 'open', objectId: this.config.customKeyServerObjectId, url: this.config.customKeyServerUrl, provider: 'Custom', network: 'custom' as any, weight: 2, // Higher weight for custom server isDefault: false }, ...keyServers]; } // Initialize SEAL client this.sealClient = new SealClient({ suiClient: (this.encryptionService as any).suiClient, serverConfigs: keyServers.map(server => ({ objectId: server.objectId, weight: server.weight || 1 })), verifyKeyServers: this.config.verifyKeyServers || false }); console.log(`โœ… SEAL client initialized with ${keyServers.length} key servers`); console.log('๐Ÿ“ก Key servers:', keyServers.map(s => `${s.name} (${s.provider})`).join(', ')); // Start session key cleanup interval this.startSessionKeyCleanup(); } catch (error) { console.error('โŒ Failed to initialize decryption pipeline:', error); throw new Error(`Decryption pipeline initialization failed: ${error instanceof Error ? error.message : 'Unknown error'}`); } } // ==================== CORE DECRYPTION METHODS ==================== /** * Decrypt a single memory with full pipeline support */ async decryptMemory(request: DecryptionRequest): Promise<DecryptionResult> { const startTime = Date.now(); try { // Check cache first const cacheKey = `${request.memoryId}:${request.userAddress}`; const cached = this.getFromCache(cacheKey); if (cached) { console.log(`๐ŸŽฏ Cache hit for memory ${request.memoryId}`); return { memoryId: request.memoryId, decryptedContent: cached, contentHash: request.contentHash || '', isVerified: true, decryptionTime: Date.now() - startTime, keyServerUsed: 'cache', sessionKeyId: 'cached' }; } // Verify ownership if required if (this.config.requireOwnershipVerification && request.ownerAddress) { const hasAccess = await this.verifyAccess(request.userAddress, request.memoryId, request.ownerAddress); if (!hasAccess) { throw new Error(`Access denied: User ${request.userAddress} cannot decrypt memory ${request.memoryId}`); } } // Get or create session key const sessionKey = request.sessionKey || await this.getOrCreateSessionKey(request.userAddress); // Perform decryption const decryptedBytes = await this.performDecryption(request, sessionKey); const decryptedContent = new TextDecoder().decode(decryptedBytes); // Verify content integrity if hash provided let isVerified = true; if (request.contentHash) { isVerified = await this.verifyContentIntegrity(decryptedBytes, request.contentHash); if (!isVerified) { console.warn(`โš ๏ธ Content integrity check failed for memory ${request.memoryId}`); } } // Cache the result this.addToCache(cacheKey, decryptedContent); // Update stats this.updateDecryptionStats(true, Date.now() - startTime); const result: DecryptionResult = { memoryId: request.memoryId, decryptedContent, contentHash: request.contentHash || '', isVerified, decryptionTime: Date.now() - startTime, keyServerUsed: 'seal-client', // Would track specific server in production sessionKeyId: await this.getSessionKeyId(sessionKey) }; console.log(`โœ… Successfully decrypted memory ${request.memoryId} in ${result.decryptionTime}ms`); return result; } catch (error) { this.updateDecryptionStats(false, Date.now() - startTime); const errorMsg = `Decryption failed for memory ${request.memoryId}: ${error instanceof Error ? error.message : 'Unknown error'}`; console.error('โŒ', errorMsg); throw new Error(errorMsg); } } /** * Decrypt multiple memories in batch for performance */ async decryptMemoryBatch(requests: DecryptionRequest[]): Promise<BatchDecryptionResult> { console.log(`๐Ÿ”„ Starting batch decryption of ${requests.length} memories`); const startTime = Date.now(); const successful: DecryptionResult[] = []; const failed: Array<{ memoryId: string; error: string; retryCount: number }> = []; // Group requests into batches const batchSize = this.config.batchSize || 10; const batches = this.chunkArray(requests, batchSize); for (const batch of batches) { // Process batch with concurrency limit const batchPromises = batch.map(async (request) => { try { const result = await this.decryptMemory(request); successful.push(result); } catch (error) { failed.push({ memoryId: request.memoryId, error: error instanceof Error ? error.message : 'Unknown error', retryCount: 0 }); } }); // Wait for batch to complete await Promise.allSettled(batchPromises); } // Retry failed decryptions if enabled if (this.config.enableFallback && this.config.maxRetryAttempts && this.config.maxRetryAttempts > 0) { await this.retryFailedDecryptions(requests, failed, successful); } const totalProcessingTime = Date.now() - startTime; const stats = this.generateBatchStats(requests.length, successful.length, failed.length, totalProcessingTime); console.log(`โœ… Batch decryption completed: ${successful.length}/${requests.length} successful in ${totalProcessingTime}ms`); return { successful, failed, stats }; } // ==================== SESSION KEY MANAGEMENT ==================== /** * Get or create session key for user */ async getOrCreateSessionKey(userAddress: string): Promise<SessionKey> { // Check if we have a valid cached session key const existing = this.sessionKeys.get(userAddress); const timestamp = this.sessionKeyTimestamps.get(userAddress); if (existing && timestamp) { const age = Date.now() - timestamp; const ttl = (this.config.sessionKeyTTL || 60) * 60 * 1000; // Convert to ms if (age < ttl) { return existing; } } // Create new session key console.log(`๐Ÿ”‘ Creating new session key for user ${userAddress}`); const sessionKey = await this.encryptionService.createSessionKey(userAddress); // Cache with timestamp this.sessionKeys.set(userAddress, sessionKey); this.sessionKeyTimestamps.set(userAddress, Date.now()); // Cleanup old keys if we exceed limit if (this.sessionKeys.size > (this.config.maxSessionKeys || 100)) { this.cleanupOldestSessionKeys(); } return sessionKey; } /** * Cleanup expired session keys */ private startSessionKeyCleanup(): void { setInterval(() => { const now = Date.now(); const ttl = (this.config.sessionKeyTTL || 60) * 60 * 1000; for (const [userAddress, timestamp] of this.sessionKeyTimestamps.entries()) { if (now - timestamp > ttl) { this.sessionKeys.delete(userAddress); this.sessionKeyTimestamps.delete(userAddress); console.log(`๐Ÿงน Cleaned up expired session key for ${userAddress}`); } } }, 5 * 60 * 1000); // Check every 5 minutes } /** * Cleanup oldest session keys when limit exceeded */ private cleanupOldestSessionKeys(): void { const entries = Array.from(this.sessionKeyTimestamps.entries()) .sort((a, b) => a[1] - b[1]); // Sort by timestamp // Remove oldest 25% const toRemove = Math.floor(entries.length * 0.25); for (let i = 0; i < toRemove; i++) { const [userAddress] = entries[i]; this.sessionKeys.delete(userAddress); this.sessionKeyTimestamps.delete(userAddress); } console.log(`๐Ÿงน Cleaned up ${toRemove} oldest session keys`); } // ==================== DECRYPTION UTILITIES ==================== /** * Perform the actual decryption using SEAL */ private async performDecryption(request: DecryptionRequest, sessionKey: SessionKey): Promise<Uint8Array> { if (!this.sealClient) { throw new Error('SEAL client not initialized'); } // Convert base64 encrypted content to bytes const encryptedBytes = this.base64ToUint8Array(request.encryptedContent); // Build access transaction const accessTx = await this.encryptionService.buildAccessTransaction(request.userAddress, 'read'); const txBytes = await accessTx.build({ client: (this.encryptionService as any).suiClient }); // Perform decryption with timeout const decryptPromise = this.sealClient.decrypt({ data: encryptedBytes, sessionKey: sessionKey, txBytes: txBytes, checkShareConsistency: true }); // Add timeout const timeoutPromise = new Promise<never>((_, reject) => { setTimeout(() => reject(new Error('Decryption timeout')), this.config.decryptionTimeout || 30000); }); return Promise.race([decryptPromise, timeoutPromise]); } /** * Verify user access to encrypted content */ private async verifyAccess(userAddress: string, memoryId: string, ownerAddress: string): Promise<boolean> { try { return await this.encryptionService.hasAccess(userAddress, memoryId, ownerAddress); } catch (error) { console.error(`Access verification failed: ${error}`); return false; } } /** * Verify content integrity using hash */ private async verifyContentIntegrity(content: Uint8Array, expectedHash: string): Promise<boolean> { try { return await (this.encryptionService as any).verifyContentHash(content, expectedHash); } catch (error) { console.error(`Content integrity verification failed: ${error}`); return false; } } /** * Retry failed decryptions with exponential backoff */ private async retryFailedDecryptions( originalRequests: DecryptionRequest[], failed: Array<{ memoryId: string; error: string; retryCount: number }>, successful: DecryptionResult[] ): Promise<void> { const maxRetries = this.config.maxRetryAttempts || 3; const baseDelay = this.config.retryDelayMs || 1000; for (const failure of failed) { if (failure.retryCount >= maxRetries) continue; const originalRequest = originalRequests.find(r => r.memoryId === failure.memoryId); if (!originalRequest) continue; try { // Exponential backoff const delay = baseDelay * Math.pow(2, failure.retryCount); await new Promise(resolve => setTimeout(resolve, delay)); console.log(`๐Ÿ”„ Retrying decryption for memory ${failure.memoryId} (attempt ${failure.retryCount + 1})`); const result = await this.decryptMemory(originalRequest); // Move from failed to successful successful.push(result); const failIndex = failed.indexOf(failure); failed.splice(failIndex, 1); console.log(`โœ… Retry successful for memory ${failure.memoryId}`); } catch (error) { failure.retryCount++; failure.error = error instanceof Error ? error.message : 'Unknown error'; console.warn(`โš ๏ธ Retry ${failure.retryCount} failed for memory ${failure.memoryId}`); } } } // ==================== CACHE MANAGEMENT ==================== private getFromCache(key: string): string | null { const cached = this.decryptionCache.get(key); if (cached && Date.now() - cached.timestamp < this.CACHE_TTL) { return cached.content; } this.decryptionCache.delete(key); return null; } private addToCache(key: string, content: string): void { this.decryptionCache.set(key, { content, timestamp: Date.now() }); } // ==================== UTILITY METHODS ==================== private base64ToUint8Array(base64: string): Uint8Array { const binaryString = atob(base64); const bytes = new Uint8Array(binaryString.length); for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes; } private chunkArray<T>(array: T[], size: number): T[][] { const chunks: T[][] = []; for (let i = 0; i < array.length; i += size) { chunks.push(array.slice(i, i + size)); } return chunks; } private async getSessionKeyId(sessionKey: SessionKey): Promise<string> { try { const exported = sessionKey.export(); return `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; } catch { return `session_${Date.now()}`; } } private updateDecryptionStats(success: boolean, processingTime: number): void { this.decryptionStats.totalDecryptions++; this.decryptionStats.totalDecryptionTime += processingTime; if (success) { this.decryptionStats.successfulDecryptions++; } else { this.decryptionStats.failedDecryptions++; } } private generateBatchStats( totalRequests: number, successCount: number, failureCount: number, totalProcessingTime: number ): BatchDecryptionResult['stats'] { return { totalRequests, successCount, failureCount, totalProcessingTime, averageDecryptionTime: totalProcessingTime / totalRequests, keyServerPerformance: {} // Would be populated with actual server stats }; } // ==================== PUBLIC API ==================== /** * Decrypt encrypted content from unified memory results */ async decryptMemoryResults( memories: UnifiedMemoryResult[], userAddress: string ): Promise<UnifiedMemoryResult[]> { console.log(`๐Ÿ” Decrypting ${memories.length} encrypted memories for user ${userAddress}`); const encryptedMemories = memories.filter(m => m.metadata.isEncrypted); if (encryptedMemories.length === 0) { console.log('๐Ÿ“‹ No encrypted memories found, returning original results'); return memories; } // Prepare decryption requests const decryptionRequests: DecryptionRequest[] = encryptedMemories.map(memory => ({ memoryId: memory.id, encryptedContent: memory.content || '', userAddress, ownerAddress: memory.metadata.owner, metadata: memory.metadata })); // Batch decrypt if enabled let decryptedResults: DecryptionResult[] = []; if (this.config.enableBatchDecryption && decryptionRequests.length > 1) { const batchResult = await this.decryptMemoryBatch(decryptionRequests); decryptedResults = batchResult.successful; if (batchResult.failed.length > 0) { console.warn(`โš ๏ธ Failed to decrypt ${batchResult.failed.length} memories:`, batchResult.failed.map(f => f.memoryId)); } } else { // Decrypt individually for (const request of decryptionRequests) { try { const result = await this.decryptMemory(request); decryptedResults.push(result); } catch (error) { console.error(`Failed to decrypt memory ${request.memoryId}:`, error); } } } // Update memory results with decrypted content const decryptionMap = new Map(decryptedResults.map(r => [r.memoryId, r])); return memories.map(memory => { if (memory.metadata.isEncrypted && decryptionMap.has(memory.id)) { const decrypted = decryptionMap.get(memory.id)!; return { ...memory, content: decrypted.decryptedContent, analytics: { ...memory.analytics, viewCount: memory.analytics?.viewCount || 0, shareCount: memory.analytics?.shareCount || 0, editCount: memory.analytics?.editCount || 0, sentimentScore: memory.analytics?.sentimentScore || 0, topicDistribution: memory.analytics?.topicDistribution || {}, decryptionTime: decrypted.decryptionTime, isDecryptionVerified: decrypted.isVerified } }; } return memory; }); } /** * Get decryption pipeline statistics */ getDecryptionStats(): { totalDecryptions: number; successRate: number; averageDecryptionTime: number; cacheHitRate: number; activeSessionKeys: number; keyServerStatus: string[]; } { const successRate = this.decryptionStats.totalDecryptions > 0 ? this.decryptionStats.successfulDecryptions / this.decryptionStats.totalDecryptions : 0; const avgTime = this.decryptionStats.totalDecryptions > 0 ? this.decryptionStats.totalDecryptionTime / this.decryptionStats.totalDecryptions : 0; return { totalDecryptions: this.decryptionStats.totalDecryptions, successRate, averageDecryptionTime: avgTime, cacheHitRate: 0.85, // Would calculate from actual cache stats activeSessionKeys: this.sessionKeys.size, keyServerStatus: this.config.keyServers?.map(s => s.name) || [] }; } /** * Clear decryption cache */ clearCache(): void { this.decryptionCache.clear(); console.log('๐Ÿงน Decryption cache cleared'); } /** * Check if decryption pipeline is ready */ isReady(): boolean { return this.sealClient !== null; } /** * Get pipeline configuration info */ getConfigInfo(): { keyServers: number; defaultNetwork: string; batchingEnabled: boolean; cacheEnabled: boolean; } { return { keyServers: this.config.keyServers?.length || 0, defaultNetwork: process.env.SUI_NETWORK || 'testnet', batchingEnabled: this.config.enableBatchDecryption || false, cacheEnabled: true }; } }