@joystick.js/db-canary/tests/performance/isolated_5000000_test.js

JoystickDB - A minimalist database server for the Joystick framework

import { rmSync, existsSync } from 'fs';
import { initialize_database, cleanup_database } from '../../src/server/lib/query_engine.js';
import { bulk_insert_with_metrics } from '../../src/server/lib/bulk_insert_optimizer.js';

const TEST_DB_PATH = './test_data/isolated_5000000_test';
const TEST_DATABASE = 'isolated_db_5000000';
const TEST_COLLECTION = 'isolated_collection';

// Generate minimal test documents
const generate_documents = (count) => {
  const documents = [];
  const test_id = Date.now().toString(36);
  
  for (let i = 0; i < count; i++) {
    documents.push({
      _id: `iso_${test_id}_${i.toString().padStart(8, '0')}`,
      idx: i,
      cat: i % 50,
      val: i % 1000,
      ts: Date.now() + i
    });
  }
  
  return documents;
};

// Aggressive memory management
const force_cleanup = async () => {
  await cleanup_database(true);
  
  // Force multiple GC cycles
  if (global.gc) {
    for (let i = 0; i < 5; i++) {
      global.gc();
      await new Promise(resolve => setTimeout(resolve, 50));
    }
  }
  
  // Wait for LMDB resources to be released
  await new Promise(resolve => setTimeout(resolve, 200));
};

// Main test execution
const run_test = async () => {
  try {
    console.log('🚀 Starting 5M Document Enterprise Scale Test (5,000,000 documents)');
    
    // Clean setup
    if (existsSync(TEST_DB_PATH)) {
      rmSync(TEST_DB_PATH, { recursive: true, force: true });
    }
    
    // Initial memory state
    const initial_memory = process.memoryUsage();
    console.log(`Initial Memory: ${Math.round(initial_memory.heapUsed / (1024 * 1024))}MB heap used`);
    
    initialize_database(TEST_DB_PATH);
    
    // Generate documents
    console.log('Generating documents...');
    const documents = generate_documents(5000000);
    
    // Run test with optimal settings for isolation
    const start_time = Date.now();
    const result = await bulk_insert_with_metrics(TEST_DATABASE, TEST_COLLECTION, documents, {
      disable_indexing: true,
      pre_allocate_map_size: true,
      sort_keys: true,
      stream_processing: true,
      batch_size: 250 // Smaller batches for very large datasets
    });
    
    const total_duration = Date.now() - start_time;
    const duration_seconds = total_duration / 1000;
    
    // Output results in parseable format
    console.log(`\n✅ 5M DOCUMENT ENTERPRISE SCALE TEST RESULTS:`);
    console.log(`Duration: ${duration_seconds.toFixed(2)} seconds`);
    console.log(`Throughput: ${result.performance.documents_per_second.toLocaleString()} docs/sec`);
    console.log(`Memory Delta: ${result.performance.memory_usage.delta_heap_mb}MB`);
    console.log(`Peak Memory: ${result.performance.memory_usage.peak_heap_mb}MB`);
    console.log(`Success Rate: 100%`);
    
    // Validate results
    if (!result.acknowledged) {
      throw new Error('Insert not acknowledged');
    }
    if (result.inserted_count !== 5000000) {
      throw new Error(`Expected ${5000000} inserts, got ${result.inserted_count}`);
    }
    
    // Performance validation
    const max_duration = 180;
    const min_throughput = 25000;
    const max_memory = 2048;
    
    if (duration_seconds > max_duration) {
      throw new Error(`Duration ${duration_seconds}s exceeds ${max_duration}s limit`);
    }
    if (result.performance.documents_per_second < min_throughput) {
      throw new Error(`Throughput ${result.performance.documents_per_second} below ${min_throughput} docs/sec target`);
    }
    if (result.performance.memory_usage.peak_heap_mb > max_memory) {
      throw new Error(`Memory ${result.performance.memory_usage.peak_heap_mb}MB exceeds ${max_memory}MB limit`);
    }
    
    console.log(`\n📈 5M DOCUMENT ENTERPRISE SCALE TEST VALIDATION:`);
    console.log(`✅ Performance targets met`);
    console.log(`✅ Memory usage within limits`);
    console.log(`✅ All ${5000000} documents inserted successfully`);
    
    // Cleanup
    await force_cleanup();
    
    const final_memory = process.memoryUsage();
    console.log(`Final Memory: ${Math.round(final_memory.heapUsed / (1024 * 1024))}MB heap used`);
    
    console.log('\n🎉 Test completed successfully');
    process.exit(0);
    
  } catch (error) {
    console.error(`\n❌ Test failed: ${error.message}`);
    console.error(error.stack);
    
    try {
      await force_cleanup();
    } catch (cleanupError) {
      console.error('Cleanup error:', cleanupError.message);
    }
    
    process.exit(1);
  }
};

// Handle process signals
process.on('SIGTERM', async () => {
  console.log('Received SIGTERM, cleaning up...');
  try {
    await force_cleanup();
  } catch (error) {
    console.error('Cleanup error:', error.message);
  }
  process.exit(1);
});

process.on('SIGINT', async () => {
  console.log('Received SIGINT, cleaning up...');
  try {
    await force_cleanup();
  } catch (error) {
    console.error('Cleanup error:', error.message);
  }
  process.exit(1);
});

// Add uncaught exception handlers
process.on('uncaughtException', async (error) => {
  console.error('\n💥 Uncaught Exception:', error.message);
  console.error(error.stack);
  
  try {
    await force_cleanup();
  } catch (cleanupError) {
    console.error('Cleanup error:', cleanupError.message);
  }
  
  process.exit(1);
});

process.on('unhandledRejection', async (reason, promise) => {
  console.error('\n💥 Unhandled Rejection at:', promise, 'reason:', reason);
  
  try {
    await force_cleanup();
  } catch (cleanupError) {
    console.error('Cleanup error:', cleanupError.message);
  }
  
  process.exit(1);
});

// Run the test
run_test();