aicf-core/examples/07-integration-vector-db.js

Universal AI Context Format (AICF) - Enterprise-grade AI memory infrastructure with 95.5% compression and zero semantic loss

/**
 * AICF v3.1 Integration Example: Vector Database
 * 
 * This example demonstrates how to integrate AICF v3.1 memory management
 * with vector databases for semantic search and retrieval.
 * 
 * Supported Vector Databases:
 * - Pinecone
 * - Weaviate
 * - Qdrant
 * - Chroma
 * 
 * Based on Google ADK patterns for production-scale semantic retrieval.
 */

const fs = require('fs').promises;
const path = require('path');

// ===== MOCK VECTOR DATABASE CLIENT =====
// In production, replace with actual vector DB client (Pinecone, Weaviate, etc.)

class MockVectorDB {
  constructor(config) {
    this.config = config;
    this.vectors = new Map();
    console.log(`✅ Connected to ${config.provider} vector database`);
  }

  async upsert(id, vector, metadata) {
    this.vectors.set(id, { vector, metadata });
    console.log(`✅ Upserted vector ${id} (dimension: ${vector.length})`);
  }

  async query(vector, topK = 5, threshold = 0.85) {
    // Calculate cosine similarity for all vectors
    const results = [];
    
    for (const [id, data] of this.vectors.entries()) {
      const similarity = this.cosineSimilarity(vector, data.vector);
      if (similarity >= threshold) {
        results.push({
          id,
          score: similarity,
          metadata: data.metadata
        });
      }
    }

    // Sort by similarity (descending) and return top K
    results.sort((a, b) => b.score - a.score);
    return results.slice(0, topK);
  }

  cosineSimilarity(vecA, vecB) {
    let dotProduct = 0;
    let normA = 0;
    let normB = 0;

    for (let i = 0; i < vecA.length; i++) {
      dotProduct += vecA[i] * vecB[i];
      normA += vecA[i] * vecA[i];
      normB += vecB[i] * vecB[i];
    }

    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
  }

  async delete(id) {
    this.vectors.delete(id);
    console.log(`✅ Deleted vector ${id}`);
  }
}

// ===== MOCK EMBEDDING SERVICE =====
// In production, replace with actual embedding service (OpenAI, Cohere, etc.)

class MockEmbeddingService {
  constructor(config) {
    this.config = config;
    this.dimension = config.dimension || 1536;
    console.log(`✅ Initialized ${config.model} embedding service (dimension: ${this.dimension})`);
  }

  async embed(text) {
    // Generate mock embedding (in production, call actual API)
    const vector = Array.from({ length: this.dimension }, () => Math.random() * 2 - 1);
    
    // Normalize vector
    const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
    const normalized = vector.map(val => val / norm);
    
    return normalized;
  }
}

// ===== AICF + VECTOR DB INTEGRATION =====

class AICFVectorIntegration {
  constructor(config) {
    this.aicfDir = config.aicfDir || '.aicf';
    this.vectorDB = new MockVectorDB(config.vectorDB);
    this.embeddings = new MockEmbeddingService(config.embeddings);
  }

  /**
   * Index AICF conversation into vector database
   */
  async indexConversation(conversationId, content, metadata = {}) {
    console.log(`\n📊 Indexing conversation: ${conversationId}`);

    // Generate embedding
    const vector = await this.embeddings.embed(content);

    // Store in vector DB
    await this.vectorDB.upsert(conversationId, vector, {
      ...metadata,
      content,
      indexed_at: new Date().toISOString(),
      source: 'aicf'
    });

    // Store embedding in AICF format
    await this.saveEmbedding(conversationId, vector, metadata);

    console.log(`✅ Indexed conversation ${conversationId}`);
  }

  /**
   * Save embedding to AICF file
   */
  async saveEmbedding(id, vector, metadata) {
    const embeddingContent = `@EMBEDDING:${id}
model=${this.embeddings.config.model}
dimension=${vector.length}
vector=${vector.join(',')}
indexed_at=${new Date().toISOString()}
similarity_threshold=0.85
keywords=${metadata.keywords || ''}
`;

    const filePath = path.join(this.aicfDir, 'embeddings.aicf');
    await fs.appendFile(filePath, embeddingContent + '\n');
  }

  /**
   * Semantic search across AICF conversations
   */
  async semanticSearch(query, topK = 5, threshold = 0.85) {
    console.log(`\n🔍 Semantic search: "${query}"`);

    // Generate query embedding
    const queryVector = await this.embeddings.embed(query);

    // Search vector DB
    const results = await this.vectorDB.query(queryVector, topK, threshold);

    console.log(`✅ Found ${results.length} results`);
    return results;
  }

  /**
   * Index all AICF conversations
   */
  async indexAllConversations() {
    console.log('\n📚 Indexing all AICF conversations...');

    // Read conversation log
    const logPath = path.join(this.aicfDir, 'conversation-log.aicf');
    const content = await fs.readFile(logPath, 'utf-8');

    // Parse conversations (simplified)
    const conversations = this.parseConversations(content);

    // Index each conversation
    for (const conv of conversations) {
      await this.indexConversation(conv.id, conv.content, conv.metadata);
    }

    console.log(`✅ Indexed ${conversations.length} conversations`);
  }

  /**
   * Parse AICF conversations (simplified)
   */
  parseConversations(content) {
    // In production, use proper AICF parser
    const conversations = [];
    const lines = content.split('\n');
    
    let currentConv = null;
    for (const line of lines) {
      if (line.startsWith('@CONVERSATION:')) {
        if (currentConv) {
          conversations.push(currentConv);
        }
        currentConv = {
          id: line.split(':')[1],
          content: '',
          metadata: {}
        };
      } else if (currentConv && line.trim()) {
        currentConv.content += line + ' ';
      }
    }
    
    if (currentConv) {
      conversations.push(currentConv);
    }

    return conversations;
  }
}

// ===== EXAMPLE USAGE =====

async function main() {
  console.log('🚀 AICF v3.1 + Vector Database Integration Example\n');
  console.log('Based on Google ADK patterns for semantic retrieval\n');

  // Initialize integration
  const integration = new AICFVectorIntegration({
    aicfDir: '.aicf',
    vectorDB: {
      provider: 'pinecone',
      apiKey: 'mock-api-key',
      environment: 'us-west1-gcp',
      index: 'aicf-conversations'
    },
    embeddings: {
      model: 'text-embedding-3-large',
      dimension: 1536
    }
  });

  // Example 1: Index individual conversations
  console.log('\n=== Example 1: Index Individual Conversations ===');
  
  await integration.indexConversation(
    'conv_001',
    'Discussion about microservices architecture and scalability patterns',
    {
      keywords: 'microservices|architecture|scalability',
      category: 'ARCHITECTURE',
      priority: 'HIGH'
    }
  );

  await integration.indexConversation(
    'conv_002',
    'Database optimization techniques for PostgreSQL and query performance',
    {
      keywords: 'database|postgresql|optimization',
      category: 'DATABASE',
      priority: 'MEDIUM'
    }
  );

  await integration.indexConversation(
    'conv_003',
    'Frontend React component design patterns and state management',
    {
      keywords: 'react|frontend|components',
      category: 'FRONTEND',
      priority: 'MEDIUM'
    }
  );

  // Example 2: Semantic search
  console.log('\n=== Example 2: Semantic Search ===');
  
  const results1 = await integration.semanticSearch(
    'How to scale backend services?',
    topK = 3,
    threshold = 0.7
  );

  console.log('\nSearch Results:');
  results1.forEach((result, i) => {
    console.log(`${i + 1}. ${result.id} (score: ${result.score.toFixed(3)})`);
    console.log(`   Content: ${result.metadata.content.substring(0, 60)}...`);
    console.log(`   Keywords: ${result.metadata.keywords || 'N/A'}`);
  });

  // Example 3: Find similar conversations
  console.log('\n=== Example 3: Find Similar Conversations ===');
  
  const results2 = await integration.semanticSearch(
    'database performance tuning',
    topK = 2,
    threshold = 0.7
  );

  console.log('\nSimilar Conversations:');
  results2.forEach((result, i) => {
    console.log(`${i + 1}. ${result.id} (similarity: ${(result.score * 100).toFixed(1)}%)`);
    console.log(`   ${result.metadata.content.substring(0, 80)}...`);
  });

  console.log('\n✅ Integration example completed!');
  console.log('\n📚 Next Steps:');
  console.log('1. Replace MockVectorDB with actual vector database client (Pinecone, Weaviate, etc.)');
  console.log('2. Replace MockEmbeddingService with actual embedding API (OpenAI, Cohere, etc.)');
  console.log('3. Implement proper AICF parsing with aicf-core library');
  console.log('4. Add error handling and retry logic');
  console.log('5. Implement batch indexing for large datasets');
  console.log('6. Add monitoring and observability');
}

// Run example
if (require.main === module) {
  main().catch(console.error);
}

module.exports = { AICFVectorIntegration, MockVectorDB, MockEmbeddingService };