bigbasealpha/examples/stream-processing-demo.js

Enterprise-Grade NoSQL Database System with Modular Logger & Offline HSM Security - Complete database platform with professional text-based logging, encryption, caching, indexing, JWT authentication, auto-generated REST API, real-time dashboard, and maste

import BigBaseAlpha from '../src/alpha.js';

/**
 * BigBaseAlpha Stream Processing Demo
 * Demonstrates real-time data stream processing capabilities
 */

console.log('🚀 BigBaseAlpha Stream Processing Demo');
console.log('=====================================\n');

async function runStreamProcessingDemo() {
    try {
        // Initialize BigBaseAlpha
        console.log('📊 Initializing BigBaseAlpha...');
        const bigbase = new BigBaseAlpha({
            dataPath: './demo_data',
            enableEncryption: true,
            enableAudit: true,
            enableCaching: true,
            enablePersistence: true,
            cacheSize: 1000,
            blockchain: { enabled: false }, // Disable blockchain for demo
            eventSourcing: { enabled: false }, // Disable event sourcing for demo
            redis: { enabled: false } // Disable redis for demo
        });

        console.log('✅ BigBaseAlpha initialized successfully\n');

        // Start stream processing engine
        console.log('🌊 Starting Stream Processing Engine...');
        await bigbase.startStreamProcessing();
        console.log('✅ Stream Processing Engine started\n');

        // Demo 1: Create data streams
        console.log('📋 Demo 1: Creating Data Streams');
        console.log('================================');

        const userEventsStream = await bigbase.createDataStream('user-events', {
            type: 'unbounded',
            format: 'json',
            description: 'User activity events'
        });
        console.log('✅ Created user-events stream');

        const sensorDataStream = await bigbase.createDataStream('sensor-data', {
            type: 'unbounded',
            format: 'json',
            description: 'IoT sensor readings'
        });
        console.log('✅ Created sensor-data stream');

        const transactionStream = await bigbase.createDataStream('transactions', {
            type: 'bounded',
            format: 'json',
            description: 'Financial transactions'
        });
        console.log('✅ Created transactions stream\n');

        // Demo 2: Add stream processors
        console.log('📋 Demo 2: Adding Stream Processors');
        console.log('==================================');

        const clickFilter = await bigbase.addStreamProcessor('user-events', {
            type: 'filter',
            function: (event) => event.action === 'click',
            description: 'Filter only click events'
        });
        console.log('✅ Added click filter processor');

        const temperatureProcessor = await bigbase.addStreamProcessor('sensor-data', {
            type: 'map',
            function: (event) => ({
                ...event,
                tempCelsius: (event.tempFahrenheit - 32) * 5/9,
                processed: true
            }),
            description: 'Convert temperature to Celsius'
        });
        console.log('✅ Added temperature conversion processor');

        const amountAggregator = await bigbase.addStreamProcessor('transactions', {
            type: 'aggregate',
            function: (events) => events.reduce((sum, e) => sum + e.amount, 0),
            description: 'Aggregate transaction amounts'
        });
        console.log('✅ Added transaction amount aggregator\n');

        // Demo 3: Create windowed streams
        console.log('📋 Demo 3: Creating Windowed Streams');
        console.log('===================================');

        const clickWindow = await bigbase.createStreamWindow('user-events', {
            type: 'tumbling',
            size: 10000, // 10 seconds
            trigger: 'time',
            aggregation: {
                type: 'count',
                field: 'action'
            }
        });
        console.log('✅ Created 10-second tumbling window for clicks');

        const temperatureWindow = await bigbase.createStreamWindow('sensor-data', {
            type: 'sliding',
            size: 30000, // 30 seconds
            slide: 10000, // 10 seconds
            trigger: 'time',
            aggregation: {
                type: 'avg',
                field: 'tempCelsius'
            }
        });
        console.log('✅ Created sliding window for temperature average\n');

        // Demo 4: Create continuous queries
        console.log('📋 Demo 4: Creating Continuous Queries');
        console.log('=====================================');

        const highTempQuery = await bigbase.createContinuousQuery('high-temp-alert', {
            streamId: 'sensor-data',
            condition: 'tempCelsius > 35',
            action: 'alert',
            description: 'Alert for high temperature readings'
        });
        console.log('✅ Created high temperature alert query');

        const fraudQuery = await bigbase.createContinuousQuery('fraud-detection', {
            streamId: 'transactions',
            condition: 'amount > 10000',
            action: 'flag',
            description: 'Flag potential fraud transactions'
        });
        console.log('✅ Created fraud detection query\n');

        // Demo 5: Create stream join
        console.log('📋 Demo 5: Creating Stream Join');
        console.log('==============================');

        const userTransactionJoin = await bigbase.createStreamJoin('user-events', 'transactions', {
            type: 'inner',
            condition: 'left.userId === right.userId',
            window: 60000, // 1 minute
            description: 'Join user events with transactions'
        });
        console.log('✅ Created user-transaction join\n');

        // Demo 6: Publish sample events
        console.log('📋 Demo 6: Publishing Sample Events');
        console.log('==================================');

        console.log('📤 Publishing user events...');
        for (let i = 0; i < 10; i++) {
            await bigbase.publishToStream('user-events', {
                userId: `user_${i % 3}`,
                action: ['click', 'view', 'scroll'][Math.floor(Math.random() * 3)],
                page: `/page${i % 5}`,
                timestamp: Date.now(),
                sessionId: `session_${Math.floor(i / 3)}`
            });
            
            if (i % 3 === 0) {
                await new Promise(resolve => setTimeout(resolve, 100)); // Small delay
            }
        }
        console.log('✅ Published 10 user events');

        console.log('📤 Publishing sensor data...');
        for (let i = 0; i < 15; i++) {
            await bigbase.publishToStream('sensor-data', {
                sensorId: `sensor_${i % 4}`,
                tempFahrenheit: 68 + Math.random() * 40, // 68-108°F
                humidity: 30 + Math.random() * 40, // 30-70%
                location: `zone_${i % 3}`,
                timestamp: Date.now(),
                batteryLevel: 80 + Math.random() * 20
            });
            
            if (i % 5 === 0) {
                await new Promise(resolve => setTimeout(resolve, 150)); // Small delay
            }
        }
        console.log('✅ Published 15 sensor readings');

        console.log('📤 Publishing transactions...');
        for (let i = 0; i < 8; i++) {
            await bigbase.publishToStream('transactions', {
                transactionId: `txn_${Date.now()}_${i}`,
                userId: `user_${i % 3}`,
                amount: Math.floor(Math.random() * 15000) + 100, // $100-$15100
                currency: 'USD',
                type: ['purchase', 'transfer', 'withdrawal'][Math.floor(Math.random() * 3)],
                timestamp: Date.now(),
                merchantId: `merchant_${i % 5}`
            });
            
            await new Promise(resolve => setTimeout(resolve, 200)); // Small delay
        }
        console.log('✅ Published 8 transactions\n');

        // Demo 7: Create aggregation streams
        console.log('📋 Demo 7: Creating Aggregation Streams');
        console.log('======================================');

        const clickAggregation = await bigbase.createAggregationStream('user-events', {
            type: 'count',
            field: 'action',
            windowSize: 5000 // 5 seconds
        });
        console.log('✅ Created click count aggregation stream');

        const temperatureAggregation = await bigbase.createAggregationStream('sensor-data', {
            type: 'avg',
            field: 'tempCelsius',
            windowSize: 10000 // 10 seconds
        });
        console.log('✅ Created temperature average aggregation stream');

        const amountAggregation = await bigbase.createAggregationStream('transactions', {
            type: 'sum',
            field: 'amount',
            windowSize: 15000 // 15 seconds
        });
        console.log('✅ Created transaction amount sum aggregation stream\n');

        // Demo 8: Create filtered streams
        console.log('📋 Demo 8: Creating Filtered Streams');
        console.log('===================================');

        const highValueTransactions = await bigbase.createFilteredStream('transactions', {
            condition: (transaction) => transaction.amount > 5000
        });
        console.log('✅ Created high-value transactions filter');

        const criticalAlerts = await bigbase.createFilteredStream('sensor-data', {
            condition: (data) => data.tempCelsius > 40 || data.humidity > 90
        });
        console.log('✅ Created critical sensor alerts filter\n');

        // Demo 9: Create transformed streams
        console.log('📋 Demo 9: Creating Transformed Streams');
        console.log('======================================');

        const enrichedUserEvents = await bigbase.createTransformedStream('user-events', {
            transform: (event) => ({
                ...event,
                enriched: true,
                region: event.userId.includes('0') ? 'US' : event.userId.includes('1') ? 'EU' : 'ASIA',
                eventScore: Math.random() * 100,
                processedAt: new Date().toISOString()
            })
        });
        console.log('✅ Created enriched user events stream');

        const normalizedSensorData = await bigbase.createTransformedStream('sensor-data', {
            transform: (data) => ({
                id: data.sensorId,
                temperature: Math.round(data.tempCelsius * 100) / 100,
                humidity: Math.round(data.humidity * 100) / 100,
                zone: data.location,
                status: data.tempCelsius > 35 ? 'critical' : data.tempCelsius > 25 ? 'warning' : 'normal',
                timestamp: data.timestamp,
                normalized: true
            })
        });
        console.log('✅ Created normalized sensor data stream\n');

        // Demo 10: Get analytics and performance metrics
        console.log('📋 Demo 10: Analytics and Performance');
        console.log('====================================');

        console.log('📊 Getting stream analytics...');
        const analytics = bigbase.getStreamAnalytics();
        console.log('Stream Analytics:', {
            totalStreams: analytics.totalStreams,
            totalEvents: analytics.totalEvents,
            totalProcessors: analytics.totalProcessors,
            avgProcessingTime: analytics.avgProcessingTime + 'ms'
        });

        console.log('⚡ Getting performance metrics...');
        const performance = bigbase.getStreamPerformanceMetrics();
        console.log('Performance Metrics:', {
            throughput: performance.throughput + ' events/sec',
            latency: performance.avgLatency + 'ms',
            memoryUsage: performance.memoryUsage + 'MB',
            cpuUsage: performance.cpuUsage + '%'
        });

        console.log('🔍 Getting system status...');
        const status = bigbase.getStreamSystemStatus();
        console.log('System Status:', {
            engineStatus: status.engineStatus,
            activeStreams: status.activeStreams,
            runningProcessors: status.runningProcessors,
            uptime: status.uptime + 'ms'
        });

        console.log('\n📋 Getting all streams...');
        const allStreams = bigbase.listDataStreams();
        console.log(`Found ${allStreams.length} streams:`);
        allStreams.forEach(stream => {
            console.log(`  - ${stream.id} (${stream.type}, ${stream.eventCount || 0} events)`);
        });

        console.log('\n🎉 Stream Processing Demo completed successfully!');
        console.log('===============================================');
        console.log('📊 Dashboard: Run "npm run stream-dashboard" to view the web interface');
        console.log('🌐 Mock Server: Run "npm run stream-server" to start the API server');
        console.log('');

        // Simulate some real-time processing for a few seconds
        console.log('🔄 Simulating real-time processing for 10 seconds...');
        const startTime = Date.now();
        const simulationInterval = setInterval(async () => {
            // Publish random events
            await bigbase.publishToStream('user-events', {
                userId: `user_${Math.floor(Math.random() * 5)}`,
                action: ['click', 'view', 'scroll', 'purchase'][Math.floor(Math.random() * 4)],
                page: `/page${Math.floor(Math.random() * 10)}`,
                timestamp: Date.now(),
                sessionId: `session_${Math.floor(Math.random() * 3)}`
            });

            if (Math.random() > 0.7) {
                await bigbase.publishToStream('sensor-data', {
                    sensorId: `sensor_${Math.floor(Math.random() * 6)}`,
                    tempFahrenheit: 70 + Math.random() * 30,
                    humidity: 40 + Math.random() * 30,
                    location: `zone_${Math.floor(Math.random() * 4)}`,
                    timestamp: Date.now(),
                    batteryLevel: 75 + Math.random() * 25
                });
            }
        }, 1000);

        setTimeout(async () => {
            clearInterval(simulationInterval);
            console.log('✅ Real-time simulation completed');
            
            // Final analytics
            const finalAnalytics = bigbase.getStreamAnalytics();
            console.log('\n📈 Final Analytics:');
            console.log(`   Total Events Processed: ${finalAnalytics.totalEvents}`);
            console.log(`   Average Processing Time: ${finalAnalytics.avgProcessingTime}ms`);
            console.log(`   Final Throughput: ${finalAnalytics.eventsPerSecond} events/sec`);
            
            // Stop stream processing
            await bigbase.stopStreamProcessing();
            console.log('\n🛑 Stream Processing Engine stopped');
            console.log('✨ Demo completed successfully!');
        }, 10000);

    } catch (error) {
        console.error('❌ Demo failed:', error);
        console.error(error.stack);
        process.exit(1);
    }
}

// Handle graceful shutdown
process.on('SIGINT', () => {
    console.log('\n🛑 Demo interrupted by user');
    process.exit(0);
});

process.on('uncaughtException', (error) => {
    console.error('❌ Uncaught exception:', error);
    process.exit(1);
});

process.on('unhandledRejection', (reason, promise) => {
    console.error('❌ Unhandled rejection at:', promise, 'reason:', reason);
    process.exit(1);
});

// Run the demo
runStreamProcessingDemo().catch(console.error);