ruvector-extensions/src/examples/persistence-example.ts

Advanced features for ruvector: embeddings, UI, exports, temporal tracking, and persistence

/**
 * Example usage of the Database Persistence module
 *
 * This example demonstrates all major features:
 * - Basic save/load operations
 * - Snapshot management
 * - Export/import
 * - Progress callbacks
 * - Auto-save configuration
 * - Incremental saves
 */

import { VectorDB } from 'ruvector';
import {
  DatabasePersistence,
  formatFileSize,
  formatTimestamp,
  estimateMemoryUsage,
} from '../persistence.js';

// ============================================================================
// Example 1: Basic Save and Load
// ============================================================================

async function example1_BasicSaveLoad() {
  console.log('\n=== Example 1: Basic Save and Load ===\n');

  // Create a vector database
  const db = new VectorDB({
    dimension: 384,
    metric: 'cosine',
  });

  // Add some sample vectors
  console.log('Adding sample vectors...');
  for (let i = 0; i < 1000; i++) {
    db.insert({
      id: `doc-${i}`,
      vector: Array(384).fill(0).map(() => Math.random()),
      metadata: {
        category: i % 3 === 0 ? 'A' : i % 3 === 1 ? 'B' : 'C',
        timestamp: Date.now() - i * 1000,
      },
    });
  }

  console.log(`Added ${db.stats().count} vectors`);

  // Create persistence manager
  const persistence = new DatabasePersistence(db, {
    baseDir: './data/example1',
    format: 'json',
    compression: 'gzip',
  });

  // Save database with progress tracking
  console.log('\nSaving database...');
  const savePath = await persistence.save({
    onProgress: (progress) => {
      console.log(`  [${progress.percentage}%] ${progress.message}`);
    },
  });

  console.log(`Saved to: ${savePath}`);

  // Create a new database and load the saved data
  const db2 = new VectorDB({ dimension: 384 });
  const persistence2 = new DatabasePersistence(db2, {
    baseDir: './data/example1',
  });

  console.log('\nLoading database...');
  await persistence2.load({
    path: savePath,
    verifyChecksum: true,
    onProgress: (progress) => {
      console.log(`  [${progress.percentage}%] ${progress.message}`);
    },
  });

  console.log(`Loaded ${db2.stats().count} vectors`);

  // Verify data integrity
  const original = db.get('doc-500');
  const loaded = db2.get('doc-500');
  console.log('\nData integrity check:');
  console.log('  Original metadata:', original?.metadata);
  console.log('  Loaded metadata:  ', loaded?.metadata);
  console.log('  Match:', JSON.stringify(original) === JSON.stringify(loaded) ? '✓' : '✗');
}

// ============================================================================
// Example 2: Snapshot Management
// ============================================================================

async function example2_SnapshotManagement() {
  console.log('\n=== Example 2: Snapshot Management ===\n');

  const db = new VectorDB({ dimension: 128 });
  const persistence = new DatabasePersistence(db, {
    baseDir: './data/example2',
    format: 'binary',
    compression: 'gzip',
    maxSnapshots: 5,
  });

  // Create initial data
  console.log('Creating initial dataset...');
  for (let i = 0; i < 500; i++) {
    db.insert({
      id: `v${i}`,
      vector: Array(128).fill(0).map(() => Math.random()),
    });
  }

  // Create snapshot before major changes
  console.log('\nCreating snapshot "before-update"...');
  const snapshot1 = await persistence.createSnapshot('before-update', {
    description: 'Baseline before adding new vectors',
    user: 'admin',
  });

  console.log(`Snapshot created: ${snapshot1.id}`);
  console.log(`  Name: ${snapshot1.name}`);
  console.log(`  Vectors: ${snapshot1.vectorCount}`);
  console.log(`  Size: ${formatFileSize(snapshot1.fileSize)}`);
  console.log(`  Created: ${formatTimestamp(snapshot1.timestamp)}`);

  // Make changes
  console.log('\nAdding more vectors...');
  for (let i = 500; i < 1000; i++) {
    db.insert({
      id: `v${i}`,
      vector: Array(128).fill(0).map(() => Math.random()),
    });
  }

  // Create another snapshot
  console.log('\nCreating snapshot "after-update"...');
  const snapshot2 = await persistence.createSnapshot('after-update');
  console.log(`Snapshot created: ${snapshot2.id} (${snapshot2.vectorCount} vectors)`);

  // List all snapshots
  console.log('\nAll snapshots:');
  const snapshots = await persistence.listSnapshots();
  for (const snapshot of snapshots) {
    console.log(`  ${snapshot.name}: ${snapshot.vectorCount} vectors, ${formatFileSize(snapshot.fileSize)}`);
  }

  // Restore from first snapshot
  console.log('\nRestoring from "before-update" snapshot...');
  await persistence.restoreSnapshot(snapshot1.id, {
    verifyChecksum: true,
    onProgress: (p) => console.log(`  [${p.percentage}%] ${p.message}`),
  });

  console.log(`After restore: ${db.stats().count} vectors`);

  // Delete a snapshot
  console.log('\nDeleting snapshot...');
  await persistence.deleteSnapshot(snapshot2.id);
  console.log('Snapshot deleted');
}

// ============================================================================
// Example 3: Export and Import
// ============================================================================

async function example3_ExportImport() {
  console.log('\n=== Example 3: Export and Import ===\n');

  // Create source database
  const sourceDb = new VectorDB({ dimension: 256 });
  console.log('Creating source database...');

  for (let i = 0; i < 2000; i++) {
    sourceDb.insert({
      id: `item-${i}`,
      vector: Array(256).fill(0).map(() => Math.random()),
      metadata: {
        type: 'product',
        price: Math.random() * 100,
        rating: Math.floor(Math.random() * 5) + 1,
      },
    });
  }

  const sourcePersistence = new DatabasePersistence(sourceDb, {
    baseDir: './data/example3/source',
  });

  // Export to different formats
  console.log('\nExporting to JSON...');
  await sourcePersistence.export({
    path: './data/example3/export/database.json',
    format: 'json',
    compress: false,
    includeIndex: false,
    onProgress: (p) => console.log(`  [${p.percentage}%] ${p.message}`),
  });

  console.log('\nExporting to compressed binary...');
  await sourcePersistence.export({
    path: './data/example3/export/database.bin.gz',
    format: 'binary',
    compress: true,
    includeIndex: true,
  });

  // Import into new database
  const targetDb = new VectorDB({ dimension: 256 });
  const targetPersistence = new DatabasePersistence(targetDb, {
    baseDir: './data/example3/target',
  });

  console.log('\nImporting from compressed binary...');
  await targetPersistence.import({
    path: './data/example3/export/database.bin.gz',
    format: 'binary',
    clear: true,
    verifyChecksum: true,
    onProgress: (p) => console.log(`  [${p.percentage}%] ${p.message}`),
  });

  console.log(`\nImport complete: ${targetDb.stats().count} vectors`);

  // Test a search to verify data integrity
  const sampleVector = sourceDb.get('item-100');
  if (sampleVector) {
    const results = targetDb.search({
      vector: sampleVector.vector,
      k: 1,
    });

    console.log('\nData integrity verification:');
    console.log('  Search for item-100:', results[0]?.id === 'item-100' ? '✓' : '✗');
    console.log('  Similarity score:', results[0]?.score.toFixed(4));
  }
}

// ============================================================================
// Example 4: Auto-Save and Incremental Saves
// ============================================================================

async function example4_AutoSaveIncremental() {
  console.log('\n=== Example 4: Auto-Save and Incremental Saves ===\n');

  const db = new VectorDB({ dimension: 64 });
  const persistence = new DatabasePersistence(db, {
    baseDir: './data/example4',
    format: 'json',
    compression: 'none',
    incremental: true,
    autoSaveInterval: 5000, // Auto-save every 5 seconds
    maxSnapshots: 3,
  });

  console.log('Auto-save enabled (every 5 seconds)');
  console.log('Incremental saves enabled');

  // Add initial batch
  console.log('\nAdding initial batch (500 vectors)...');
  for (let i = 0; i < 500; i++) {
    db.insert({
      id: `vec-${i}`,
      vector: Array(64).fill(0).map(() => Math.random()),
    });
  }

  // Manual incremental save
  console.log('\nPerforming initial save...');
  await persistence.save();

  // Simulate ongoing operations
  console.log('\nAdding more vectors...');
  for (let i = 500; i < 600; i++) {
    db.insert({
      id: `vec-${i}`,
      vector: Array(64).fill(0).map(() => Math.random()),
    });
  }

  // Incremental save (only saves changes)
  console.log('\nPerforming incremental save...');
  const incrementalPath = await persistence.saveIncremental();

  if (incrementalPath) {
    console.log(`Incremental save completed: ${incrementalPath}`);
  } else {
    console.log('No changes detected (skip)');
  }

  // Wait for auto-save to trigger
  console.log('\nWaiting for auto-save (5 seconds)...');
  await new Promise(resolve => setTimeout(resolve, 6000));

  // Cleanup
  console.log('\nShutting down (final save)...');
  await persistence.shutdown();
  console.log('Shutdown complete');
}

// ============================================================================
// Example 5: Advanced Progress Tracking
// ============================================================================

async function example5_AdvancedProgress() {
  console.log('\n=== Example 5: Advanced Progress Tracking ===\n');

  const db = new VectorDB({ dimension: 512 });

  // Create large dataset
  console.log('Creating large dataset (5000 vectors)...');
  const startTime = Date.now();

  for (let i = 0; i < 5000; i++) {
    db.insert({
      id: `large-${i}`,
      vector: Array(512).fill(0).map(() => Math.random()),
      metadata: {
        batch: Math.floor(i / 100),
        index: i,
      },
    });
  }

  console.log(`Dataset created in ${Date.now() - startTime}ms`);

  const persistence = new DatabasePersistence(db, {
    baseDir: './data/example5',
    format: 'binary',
    compression: 'gzip',
    batchSize: 500, // Process in batches of 500
  });

  // Custom progress handler with detailed stats
  let lastUpdate = Date.now();
  const progressHandler = (progress: any) => {
    const now = Date.now();
    const elapsed = now - lastUpdate;

    if (elapsed > 100) { // Update max every 100ms
      const bar = '█'.repeat(Math.floor(progress.percentage / 2)) +
                  '░'.repeat(50 - Math.floor(progress.percentage / 2));

      process.stdout.write(
        `\r  [${bar}] ${progress.percentage}% - ${progress.message}`.padEnd(100)
      );

      lastUpdate = now;
    }
  };

  // Save with detailed progress
  console.log('\nSaving with progress tracking:');
  const saveStart = Date.now();

  await persistence.save({
    compress: true,
    onProgress: progressHandler,
  });

  console.log(`\n\nSave completed in ${Date.now() - saveStart}ms`);

  // Load with progress
  const db2 = new VectorDB({ dimension: 512 });
  const persistence2 = new DatabasePersistence(db2, {
    baseDir: './data/example5',
  });

  console.log('\nLoading with progress tracking:');
  const loadStart = Date.now();

  await persistence2.load({
    path: './data/example5/database.bin.gz',
    verifyChecksum: true,
    onProgress: progressHandler,
  });

  console.log(`\n\nLoad completed in ${Date.now() - loadStart}ms`);
  console.log(`Loaded ${db2.stats().count} vectors`);
}

// ============================================================================
// Run All Examples
// ============================================================================

async function runAllExamples() {
  try {
    await example1_BasicSaveLoad();
    await example2_SnapshotManagement();
    await example3_ExportImport();
    await example4_AutoSaveIncremental();
    await example5_AdvancedProgress();

    console.log('\n\n✓ All examples completed successfully!\n');
  } catch (error) {
    console.error('\n✗ Error running examples:', error);
    process.exit(1);
  }
}

// Run examples if executed directly
if (import.meta.url === `file://${process.argv[1]}`) {
  runAllExamples();
}

export {
  example1_BasicSaveLoad,
  example2_SnapshotManagement,
  example3_ExportImport,
  example4_AutoSaveIncremental,
  example5_AdvancedProgress,
};