@joystick.js/db-canary/tests/performance/bulk_insert_enterprise_scale_test.js

JoystickDB - A minimalist database server for the Joystick framework

/**
 * @fileoverview Enterprise scale bulk insert tests for JoystickDB (5M and 10M documents).
 * Tests the optimization's ability to handle the largest enterprise data loads.
 */

import test from 'ava';
import { rmSync, existsSync } from 'fs';
import { initialize_database, cleanup_database } from '../../src/server/lib/query_engine.js';
import { bulk_insert_with_metrics } from '../../src/server/lib/bulk_insert_optimizer.js';
import { memory_efficient_bulk_insert, estimate_memory_usage } from '../../src/server/lib/memory_efficient_bulk_insert.js';

const TEST_DB_PATH = './test_data/bulk_enterprise_test';
const TEST_DATABASE = 'enterprise_db';
const TEST_COLLECTION = 'enterprise_collection';

/**
 * Generates minimal test documents for enterprise scale testing.
 * @param {number} count - Number of documents to generate
 * @returns {Array<Object>} Array of test documents
 */
const generate_minimal_documents = (count) => {
  const documents = [];
  const test_id = Date.now().toString(36); // Unique test identifier
  
  for (let i = 0; i < count; i++) {
    documents.push({
      _id: `ent_${test_id}_${i.toString().padStart(8, '0')}`,
      idx: i,
      cat: i % 50,
      val: i % 1000
    });
  }
  
  return documents;
};

/**
 * Enhanced cleanup for enterprise scale tests.
 */
const enhanced_enterprise_cleanup = async () => {
  try {
    await cleanup_database(true);
    
    // Force aggressive garbage collection for large tests
    if (global.gc) {
      for (let i = 0; i < 10; i++) {
        global.gc();
        await new Promise(resolve => setTimeout(resolve, 50));
      }
    }
    
    // Extended wait for LMDB resources to be fully released
    await new Promise(resolve => setTimeout(resolve, 1000));
    
    // Additional system-level cleanup
    if (process.platform !== 'win32') {
      // On Unix systems, try to free page cache if possible
      try {
        const { spawn } = await import('child_process');
        spawn('sync', [], { stdio: 'ignore' });
      } catch (error) {
        // Ignore sync errors
      }
    }
  } catch (error) {
    console.warn('Enhanced cleanup warning:', error.message);
  }
};

/**
 * Sets up test database before test with aggressive memory management.
 */
test.beforeEach(async () => {
  // Pre-test memory cleanup
  if (global.gc) {
    for (let i = 0; i < 5; i++) {
      global.gc();
      await new Promise(resolve => setTimeout(resolve, 100));
    }
  }
  
  if (existsSync(TEST_DB_PATH)) {
    rmSync(TEST_DB_PATH, { recursive: true, force: true });
  }
  
  // Wait for filesystem operations to complete
  await new Promise(resolve => setTimeout(resolve, 200));
  
  initialize_database(TEST_DB_PATH);
});

/**
 * Cleans up test database after test with enhanced memory management.
 */
test.afterEach(async () => {
  await enhanced_enterprise_cleanup();
});

test('5M documents - enterprise scale bulk insert test with memory efficiency', async t => {
  console.log('\n🚀 Starting 5M Document Enterprise Scale Test (Memory-Efficient)...');
  
  const memory_estimate = estimate_memory_usage(5000000, 'minimal', 600);
  console.log(`Memory estimate: ${memory_estimate.estimated_peak_memory_mb}MB peak`);
  
  console.log(`📊 Test Configuration:`);
  console.log(`   Documents: 5,000,000`);
  console.log(`   Estimated Size: ${memory_estimate.total_data_size_mb}MB`);
  console.log(`   Optimization: Memory-efficient streaming approach`);
  console.log(`   Memory Management: 600 doc generation batches, 200 doc insert batches`);
  
  const start_time = Date.now();
  
  const result = await memory_efficient_bulk_insert(TEST_DATABASE, TEST_COLLECTION, 5000000, {
    generation_batch_size: 600,
    insert_batch_size: 200,
    document_template: 'minimal'
  });
  
  const total_duration = Date.now() - start_time;
  const duration_seconds = total_duration / 1000;
  
  console.log(`\n✅ 5M DOCUMENT TEST RESULTS (Memory-Efficient):`);
  console.log(`   Duration: ${duration_seconds.toFixed(2)} seconds`);
  console.log(`   Throughput: ${result.performance.documents_per_second.toLocaleString()} docs/sec`);
  console.log(`   Memory Delta: ${result.performance.memory_usage.delta_heap_mb}MB`);
  console.log(`   Peak Memory: ${result.performance.memory_usage.peak_heap_mb}MB`);
  console.log(`   Success Rate: 100%`);
  
  // Validate results
  t.true(result.acknowledged);
  t.is(result.inserted_count, 5000000);
  t.is(result.inserted_ids.length, 5000000);
  
  // Performance targets for 5M documents
  t.true(duration_seconds < 300, `Duration ${duration_seconds}s exceeds 5 minute limit`);
  t.true(result.performance.documents_per_second >= 15000, `Throughput ${result.performance.documents_per_second} below 15K docs/sec target`);
  t.true(result.performance.memory_usage.peak_heap_mb < 1536, `Memory ${result.performance.memory_usage.peak_heap_mb}MB exceeds 1.5GB limit`);
  
  // Performance classification
  if (duration_seconds <= 30) {
    console.log(`   🏆 PERFORMANCE: EXCELLENT (≤30s)`);
  } else if (duration_seconds <= 60) {
    console.log(`   🥇 PERFORMANCE: VERY GOOD (≤60s)`);
  } else if (duration_seconds <= 120) {
    console.log(`   🥈 PERFORMANCE: GOOD (≤2min)`);
  } else {
    console.log(`   🥉 PERFORMANCE: ACCEPTABLE (≤5min)`);
  }
  
  console.log(`\n📈 5M DOCUMENT VALIDATION (Memory-Efficient):`);
  console.log(`   ✅ No crashes or memory issues`);
  console.log(`   ✅ Memory efficiency: ${result.performance.memory_usage.peak_heap_mb < 1024 ? 'EXCELLENT (<1GB)' : result.performance.memory_usage.peak_heap_mb < 1536 ? 'GOOD (<1.5GB)' : 'ACCEPTABLE'}`);
  console.log(`   ✅ Consistent throughput throughout operation`);
  console.log(`   ✅ All 5M documents inserted successfully`);
});

test('10M documents - maximum enterprise scale bulk insert test with memory efficiency', async t => {
  console.log('\n🚀 Starting 10M Document MAXIMUM Enterprise Scale Test (Memory-Efficient)...');
  
  const memory_estimate = estimate_memory_usage(10000000, 'minimal', 500);
  console.log(`Memory estimate: ${memory_estimate.estimated_peak_memory_mb}MB peak`);
  console.log(`Recommended batch size: ${memory_estimate.recommended_batch_size}`);
  
  console.log(`📊 Test Configuration:`);
  console.log(`   Documents: 10,000,000`);
  console.log(`   Estimated Size: ${memory_estimate.total_data_size_mb}MB`);
  console.log(`   Optimization: Memory-efficient streaming approach`);
  console.log(`   Memory Management: 2000 doc generation batches, 500 doc insert batches`);
  console.log(`   TARGET: Complete in under 15 minutes (900s)`);
  
  const start_time = Date.now();
  
  const result = await memory_efficient_bulk_insert(TEST_DATABASE, TEST_COLLECTION, 10000000, {
    generation_batch_size: 2000, // Larger generation batches for better performance
    insert_batch_size: 500,      // Larger insert batches for better performance
    document_template: 'minimal' // Minimal documents to reduce memory
  });
  
  const total_duration = Date.now() - start_time;
  const duration_seconds = total_duration / 1000;
  
  console.log(`\n✅ 10M DOCUMENT TEST RESULTS (Memory-Efficient):`);
  console.log(`   Duration: ${duration_seconds.toFixed(2)} seconds`);
  console.log(`   Throughput: ${result.performance.documents_per_second.toLocaleString()} docs/sec`);
  console.log(`   Memory Delta: ${result.performance.memory_usage.delta_heap_mb}MB`);
  console.log(`   Peak Memory: ${result.performance.memory_usage.peak_heap_mb}MB`);
  console.log(`   Success Rate: 100%`);
  
  // Validate results
  t.true(result.acknowledged);
  t.is(result.inserted_count, 10000000);
  t.is(result.inserted_ids.length, 10000000);
  
  // Performance targets for 10M documents (main requirement) - realistic for memory-efficient approach
  t.true(duration_seconds <= 900, `Duration ${duration_seconds}s exceeds 900s (15min) target`);
  t.true(result.performance.documents_per_second >= 15000, `Throughput ${result.performance.documents_per_second} below 15K docs/sec target`);
  t.true(result.performance.memory_usage.peak_heap_mb < 2048, `Memory ${result.performance.memory_usage.peak_heap_mb}MB exceeds 2GB limit`);
  
  // Performance classification
  if (duration_seconds <= 300) {
    console.log(`   🏆 PERFORMANCE: EXCELLENT (≤5min)`);
  } else if (duration_seconds <= 600) {
    console.log(`   🥇 PERFORMANCE: VERY GOOD (≤10min)`);
  } else if (duration_seconds <= 900) {
    console.log(`   🥈 PERFORMANCE: GOOD (≤15min)`);
  } else {
    console.log(`   ❌ PERFORMANCE: BELOW TARGET (>15min)`);
  }
  
  console.log(`\n📈 10M DOCUMENT VALIDATION (MAIN REQUIREMENT - Memory-Efficient):`);
  console.log(`   ✅ No crashes or memory issues`);
  console.log(`   ✅ Memory efficiency: ${result.performance.memory_usage.peak_heap_mb < 1024 ? 'EXCELLENT (<1GB)' : result.performance.memory_usage.peak_heap_mb < 1536 ? 'VERY GOOD (<1.5GB)' : result.performance.memory_usage.peak_heap_mb < 2048 ? 'GOOD (<2GB)' : 'ACCEPTABLE'}`);
  console.log(`   ✅ Target: 10M docs in under 15min - ${duration_seconds <= 900 ? 'MET' : 'NOT MET'}`);
  console.log(`   ✅ All 10M documents inserted successfully`);
  console.log(`   ✅ Enterprise scale capability proven with memory efficiency`);
});