defarm-sdk/lib/queue-manager.js

DeFarm SDK - On-premise blockchain data processing and tokenization engine for agriculture supply chain

const { EventEmitter } = require('events');
const crypto = require('crypto');

/**
 * Queue Manager for DeFarm SDK
 * Robust job queue system with retry, priority, and error handling
 */
class QueueManager extends EventEmitter {
  constructor(config = {}) {
    super();
    
    this.config = {
      name: config.name || 'defarm-queue',
      workers: config.workers || 4,
      maxRetries: config.maxRetries || 3,
      retryDelay: config.retryDelay || 5000,
      timeout: config.timeout || 300000, // 5 minutes
      batchSize: config.batchSize || 100,
      persistence: config.persistence !== false,
      redis: config.redis || null,
      database: config.database || null,
      cleanupInterval: config.cleanupInterval || 3600000, // 1 hour
      deadLetterQueue: config.deadLetterQueue !== false,
      metrics: config.metrics !== false
    };

    // Queue state
    this.jobs = new Map();
    this.workers = new Map();
    this.processors = new Map();
    this.middleware = [];
    this.isRunning = false;
    this.isPaused = false;
    
    // Statistics
    this.stats = {
      totalJobs: 0,
      completedJobs: 0,
      failedJobs: 0,
      retriedJobs: 0,
      activeJobs: 0,
      waitingJobs: 0,
      delayedJobs: 0,
      processingTime: [],
      errorRate: 0,
      throughput: 0
    };

    // Dead letter queue for failed jobs
    this.deadLetterQueue = new Map();
    
    // Cleanup timer
    this.cleanupTimer = null;
    
    // Performance monitoring
    this.performanceMonitor = {
      startTime: Date.now(),
      jobsProcessed: 0,
      totalProcessingTime: 0
    };
  }

  /**
   * Initialize queue manager
   */
  async initialize() {
    console.log('🚀 Initializing Queue Manager...');

    // Setup persistence if configured
    if (this.config.persistence) {
      await this.setupPersistence();
    }

    // Register default processors
    this.registerDefaultProcessors();

    // Start cleanup timer
    this.startCleanupTimer();

    // Handle graceful shutdown
    process.on('SIGTERM', () => this.shutdown());
    process.on('SIGINT', () => this.shutdown());

    console.log('✅ Queue Manager initialized');
  }

  /**
   * Setup persistence layer
   */
  async setupPersistence() {
    if (this.config.redis) {
      // Setup Redis persistence
      try {
        const Redis = require('redis');
        this.redis = Redis.createClient(this.config.redis);
        await this.redis.connect();
        console.log('  ✓ Redis persistence enabled');
      } catch (error) {
        console.warn('  ⚠ Redis not available, using in-memory queue');
      }
    } else if (this.config.database) {
      // Use database for persistence
      this.database = this.config.database;
      console.log('  ✓ Database persistence enabled');
    }
  }

  /**
   * Register default processors
   */
  registerDefaultProcessors() {
    // Agriculture data processor
    this.registerProcessor('process-agriculture', async (job) => {
      const { sdk } = require('../index');
      const result = await sdk.processAgricultureData(job.data, {
        async: false
      });
      return result;
    });

    // Tokenization processor
    this.registerProcessor('tokenize-asset', async (job) => {
      const { sdk } = require('../index');
      const result = await sdk.createAssetToken(job.data.asset, job.data.options);
      return result;
    });

    // Blockchain recording processor
    this.registerProcessor('blockchain-record', async (job) => {
      const { sdk } = require('../index');
      if (!sdk.blockchainEngine) {
        throw new Error('Blockchain engine not initialized');
      }
      const result = await sdk.blockchainEngine.recordData(job.data);
      return result;
    });

    // Batch processing processor
    this.registerProcessor('batch-process', async (job) => {
      const results = [];
      const errors = [];
      
      for (const item of job.data.items) {
        try {
          const result = await this.processItem(item, job.data.processor);
          results.push(result);
          
          // Update progress
          const progress = (results.length / job.data.items.length) * 100;
          await this.updateJobProgress(job.id, progress);
        } catch (error) {
          errors.push({
            item,
            error: error.message
          });
        }
      }
      
      return {
        processed: results.length,
        failed: errors.length,
        results,
        errors
      };
    });
  }

  /**
   * Add job to queue
   */
  async addJob(type, data, options = {}) {
    const job = {
      id: options.id || this.generateJobId(),
      type,
      data,
      priority: options.priority || 0,
      delay: options.delay || 0,
      timeout: options.timeout || this.config.timeout,
      maxRetries: options.maxRetries || this.config.maxRetries,
      backoff: options.backoff || 'exponential',
      status: 'waiting',
      attempts: 0,
      createdAt: Date.now(),
      scheduledFor: Date.now() + (options.delay || 0),
      metadata: options.metadata || {},
      progress: 0,
      logs: [],
      dependencies: options.dependencies || [],
      result: null,
      error: null
    };

    // Validate job
    this.validateJob(job);

    // Apply middleware
    for (const mw of this.middleware) {
      job = await mw.before(job);
    }

    // Store job
    this.jobs.set(job.id, job);
    this.stats.totalJobs++;
    this.stats.waitingJobs++;

    // Persist if configured
    if (this.config.persistence) {
      await this.persistJob(job);
    }

    // Handle delayed jobs
    if (job.delay > 0) {
      this.stats.delayedJobs++;
      this.scheduleDelayedJob(job);
    } else {
      this.emit('job:added', job);
      this.processNextJob();
    }

    console.log(`📋 Job added: ${job.type} (${job.id}) Priority: ${job.priority}`);
    
    return job;
  }

  /**
   * Add multiple jobs
   */
  async addBatch(jobs) {
    const results = [];
    
    for (const jobConfig of jobs) {
      const job = await this.addJob(
        jobConfig.type,
        jobConfig.data,
        jobConfig.options
      );
      results.push(job);
    }
    
    return results;
  }

  /**
   * Register job processor
   */
  registerProcessor(type, processor, options = {}) {
    console.log(`⚙️ Registering processor: ${type}`);
    
    this.processors.set(type, {
      handler: processor,
      concurrency: options.concurrency || 1,
      timeout: options.timeout || this.config.timeout,
      retries: options.retries || this.config.maxRetries
    });
    
    return this;
  }

  /**
   * Add middleware
   */
  use(middleware) {
    this.middleware.push({
      before: middleware.before || ((job) => job),
      after: middleware.after || ((job) => job),
      error: middleware.error || ((job, error) => { throw error; })
    });
    
    return this;
  }

  /**
   * Start processing jobs
   */
  start() {
    if (this.isRunning) return;
    
    console.log(`🚀 Starting queue processing with ${this.config.workers} workers`);
    this.isRunning = true;
    this.isPaused = false;
    
    // Start workers
    for (let i = 0; i < this.config.workers; i++) {
      this.startWorker(i);
    }
    
    this.emit('queue:started');
  }

  /**
   * Pause processing
   */
  pause() {
    console.log('⏸️ Pausing queue processing');
    this.isPaused = true;
    this.emit('queue:paused');
  }

  /**
   * Resume processing
   */
  resume() {
    console.log('▶️ Resuming queue processing');
    this.isPaused = false;
    this.processNextJob();
    this.emit('queue:resumed');
  }

  /**
   * Stop processing
   */
  async stop() {
    console.log('🛑 Stopping queue processing...');
    this.isRunning = false;
    
    // Wait for active jobs
    await this.waitForActiveJobs();
    
    this.emit('queue:stopped');
    console.log('✅ Queue stopped');
  }

  /**
   * Start a worker
   */
  startWorker(workerId) {
    const worker = {
      id: workerId,
      status: 'idle',
      currentJob: null,
      processedJobs: 0,
      errors: 0,
      startTime: Date.now()
    };
    
    this.workers.set(workerId, worker);
    
    const processLoop = async () => {
      while (this.isRunning) {
        if (this.isPaused) {
          await this.sleep(1000);
          continue;
        }
        
        try {
          const job = await this.getNextJob();
          
          if (job) {
            worker.status = 'busy';
            worker.currentJob = job.id;
            
            await this.processJob(job, worker);
            
            worker.processedJobs++;
            worker.currentJob = null;
            worker.status = 'idle';
          } else {
            await this.sleep(100);
          }
        } catch (error) {
          console.error(`Worker ${workerId} error:`, error);
          worker.errors++;
          worker.status = 'idle';
          await this.sleep(5000);
        }
      }
    };
    
    processLoop();
    console.log(`👷 Worker ${workerId} started`);
  }

  /**
   * Get next job to process
   */
  async getNextJob() {
    const now = Date.now();
    let highestPriority = -Infinity;
    let nextJob = null;
    
    // Check for jobs with satisfied dependencies
    for (const job of this.jobs.values()) {
      if (job.status === 'waiting' && 
          job.scheduledFor <= now &&
          this.areDependenciesSatisfied(job)) {
        
        if (job.priority > highestPriority) {
          highestPriority = job.priority;
          nextJob = job;
        }
      }
    }
    
    if (nextJob) {
      nextJob.status = 'active';
      nextJob.startedAt = Date.now();
      
      this.stats.waitingJobs--;
      this.stats.activeJobs++;
      
      this.emit('job:started', nextJob);
    }
    
    return nextJob;
  }

  /**
   * Process a job
   */
  async processJob(job, worker) {
    const processor = this.processors.get(job.type);
    
    if (!processor) {
      await this.failJob(job, new Error(`No processor for type: ${job.type}`));
      return;
    }
    
    try {
      console.log(`⚙️ Processing: ${job.type} (${job.id})`);
      
      // Create job context
      const context = {
        id: job.id,
        type: job.type,
        data: job.data,
        metadata: job.metadata,
        attempt: job.attempts + 1,
        log: (message) => this.addJobLog(job, message),
        progress: (percent) => this.updateJobProgress(job.id, percent),
        emit: (event, data) => this.emit(`job:${event}`, { job, data })
      };
      
      // Execute with timeout
      const result = await this.executeWithTimeout(
        processor.handler(context),
        job.timeout
      );
      
      // Apply after middleware
      for (const mw of this.middleware) {
        await mw.after(job, result);
      }
      
      await this.completeJob(job, result);
      
    } catch (error) {
      console.error(`❌ Job failed: ${job.id}`, error.message);
      
      // Apply error middleware
      for (const mw of this.middleware) {
        try {
          await mw.error(job, error);
        } catch (mwError) {
          console.error('Middleware error:', mwError);
        }
      }
      
      // Handle retry
      if (job.attempts < job.maxRetries) {
        await this.retryJob(job, error);
      } else {
        await this.failJob(job, error);
      }
    }
  }

  /**
   * Complete a job
   */
  async completeJob(job, result) {
    job.status = 'completed';
    job.result = result;
    job.completedAt = Date.now();
    job.processingTime = job.completedAt - job.startedAt;
    job.progress = 100;
    
    this.stats.activeJobs--;
    this.stats.completedJobs++;
    
    // Update performance metrics
    this.updatePerformanceMetrics(job);
    
    // Persist if configured
    if (this.config.persistence) {
      await this.persistJobUpdate(job);
    }
    
    console.log(`✅ Completed: ${job.id} (${job.processingTime}ms)`);
    this.emit('job:completed', job);
    
    // Check dependent jobs
    this.checkDependentJobs(job.id);
  }

  /**
   * Retry a job
   */
  async retryJob(job, error) {
    job.attempts++;
    job.lastError = error.message;
    job.status = 'waiting';
    
    // Calculate retry delay
    const delay = this.calculateRetryDelay(job);
    job.scheduledFor = Date.now() + delay;
    
    this.stats.activeJobs--;
    this.stats.waitingJobs++;
    this.stats.retriedJobs++;
    
    // Add to logs
    this.addJobLog(job, `Retry ${job.attempts}/${job.maxRetries}: ${error.message}`);
    
    // Persist if configured
    if (this.config.persistence) {
      await this.persistJobUpdate(job);
    }
    
    console.log(`🔄 Retry scheduled: ${job.id} (attempt ${job.attempts})`);
    this.emit('job:retry', job);
    
    // Schedule retry
    setTimeout(() => {
      this.processNextJob();
    }, delay);
  }

  /**
   * Fail a job permanently
   */
  async failJob(job, error) {
    job.status = 'failed';
    job.error = error.message;
    job.failedAt = Date.now();
    job.processingTime = job.failedAt - (job.startedAt || job.createdAt);
    
    this.stats.activeJobs--;
    this.stats.failedJobs++;
    
    // Add to dead letter queue if configured
    if (this.config.deadLetterQueue) {
      this.deadLetterQueue.set(job.id, job);
    }
    
    // Persist if configured
    if (this.config.persistence) {
      await this.persistJobUpdate(job);
    }
    
    console.log(`❌ Failed permanently: ${job.id} - ${error.message}`);
    this.emit('job:failed', job);
  }

  /**
   * Calculate retry delay
   */
  calculateRetryDelay(job) {
    const baseDelay = this.config.retryDelay;
    
    switch (job.backoff) {
      case 'exponential':
        return baseDelay * Math.pow(2, job.attempts - 1);
      case 'linear':
        return baseDelay * job.attempts;
      case 'fixed':
      default:
        return baseDelay;
    }
  }

  /**
   * Check if dependencies are satisfied
   */
  areDependenciesSatisfied(job) {
    if (!job.dependencies || job.dependencies.length === 0) {
      return true;
    }
    
    for (const depId of job.dependencies) {
      const depJob = this.jobs.get(depId);
      if (!depJob || depJob.status !== 'completed') {
        return false;
      }
    }
    
    return true;
  }

  /**
   * Check dependent jobs after completion
   */
  checkDependentJobs(completedJobId) {
    for (const job of this.jobs.values()) {
      if (job.dependencies && job.dependencies.includes(completedJobId)) {
        if (this.areDependenciesSatisfied(job)) {
          this.processNextJob();
        }
      }
    }
  }

  /**
   * Update job progress
   */
  async updateJobProgress(jobId, progress) {
    const job = this.jobs.get(jobId);
    if (!job) return;
    
    job.progress = Math.min(100, Math.max(0, progress));
    
    if (this.config.persistence) {
      await this.persistJobUpdate(job);
    }
    
    this.emit('job:progress', { job, progress: job.progress });
  }

  /**
   * Add log entry to job
   */
  addJobLog(job, message) {
    const logEntry = {
      timestamp: new Date().toISOString(),
      message
    };
    
    job.logs.push(logEntry);
    
    if (job.logs.length > 100) {
      job.logs.shift(); // Keep only last 100 logs
    }
    
    console.log(`📝 [${job.id}] ${message}`);
  }

  /**
   * Get job by ID
   */
  getJob(jobId) {
    return this.jobs.get(jobId);
  }

  /**
   * Get jobs by status
   */
  getJobs(status = null, limit = 50) {
    let jobs = Array.from(this.jobs.values());
    
    if (status) {
      jobs = jobs.filter(job => job.status === status);
    }
    
    // Sort by priority and creation time
    jobs.sort((a, b) => {
      if (a.priority !== b.priority) {
        return b.priority - a.priority;
      }
      return a.createdAt - b.createdAt;
    });
    
    return jobs.slice(0, limit);
  }

  /**
   * Remove job
   */
  async removeJob(jobId) {
    const job = this.jobs.get(jobId);
    if (!job) return false;
    
    if (job.status === 'active') {
      throw new Error('Cannot remove active job');
    }
    
    this.jobs.delete(jobId);
    
    if (this.config.persistence) {
      await this.removePersistedJob(jobId);
    }
    
    this.emit('job:removed', job);
    return true;
  }

  /**
   * Get queue statistics
   */
  getStats() {
    // Calculate additional metrics
    const now = Date.now();
    const uptime = now - this.performanceMonitor.startTime;
    const throughput = (this.stats.completedJobs / (uptime / 1000)) || 0;
    
    const avgProcessingTime = this.stats.processingTime.length > 0
      ? this.stats.processingTime.reduce((a, b) => a + b, 0) / this.stats.processingTime.length
      : 0;
    
    const errorRate = this.stats.totalJobs > 0
      ? (this.stats.failedJobs / this.stats.totalJobs) * 100
      : 0;
    
    return {
      ...this.stats,
      throughput: throughput.toFixed(2),
      avgProcessingTime: Math.round(avgProcessingTime),
      errorRate: errorRate.toFixed(2),
      uptime,
      workers: {
        total: this.workers.size,
        busy: Array.from(this.workers.values()).filter(w => w.status === 'busy').length,
        idle: Array.from(this.workers.values()).filter(w => w.status === 'idle').length
      },
      processors: Array.from(this.processors.keys()),
      deadLetterQueue: this.deadLetterQueue.size
    };
  }

  /**
   * Clean old jobs
   */
  async cleanup(olderThan = 24 * 60 * 60 * 1000) {
    const cutoff = Date.now() - olderThan;
    let cleaned = 0;
    
    for (const [jobId, job] of this.jobs) {
      if ((job.status === 'completed' || job.status === 'failed') && 
          (job.completedAt || job.failedAt) < cutoff) {
        this.jobs.delete(jobId);
        cleaned++;
      }
    }
    
    if (cleaned > 0) {
      console.log(`🧹 Cleaned ${cleaned} old jobs`);
    }
    
    return cleaned;
  }

  /**
   * Recover jobs from persistence
   */
  async recoverJobs() {
    if (!this.config.persistence) return;
    
    console.log('🔄 Recovering persisted jobs...');
    
    try {
      const jobs = await this.loadPersistedJobs();
      
      for (const job of jobs) {
        // Reset active jobs to waiting
        if (job.status === 'active') {
          job.status = 'waiting';
          job.attempts++;
        }
        
        this.jobs.set(job.id, job);
      }
      
      console.log(`  ✓ Recovered ${jobs.length} jobs`);
    } catch (error) {
      console.error('Failed to recover jobs:', error);
    }
  }

  // Persistence methods (would implement Redis/DB storage)
  
  async persistJob(job) {
    if (this.database) {
      await this.database.execute(`
        INSERT INTO ${this.database.getTableName('queue')}
        (job_id, job_type, priority, status, data, attempts, max_retries, created_at, scheduled_for)
        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
        ON CONFLICT (job_id) DO UPDATE SET
          status = EXCLUDED.status,
          data = EXCLUDED.data,
          attempts = EXCLUDED.attempts
      `, [
        job.id, job.type, job.priority, job.status,
        JSON.stringify(job.data), job.attempts, job.maxRetries,
        new Date(job.createdAt), new Date(job.scheduledFor)
      ]);
    }
  }

  async persistJobUpdate(job) {
    if (this.database) {
      await this.database.execute(`
        UPDATE ${this.database.getTableName('queue')}
        SET status = $1, attempts = $2, result = $3, error = $4,
            started_at = $5, completed_at = $6
        WHERE job_id = $7
      `, [
        job.status, job.attempts,
        job.result ? JSON.stringify(job.result) : null,
        job.error,
        job.startedAt ? new Date(job.startedAt) : null,
        job.completedAt ? new Date(job.completedAt) : null,
        job.id
      ]);
    }
  }

  async loadPersistedJobs() {
    if (this.database) {
      const result = await this.database.execute(`
        SELECT * FROM ${this.database.getTableName('queue')}
        WHERE status IN ('waiting', 'active')
        ORDER BY priority DESC, created_at ASC
      `);
      
      return result.rows.map(row => ({
        id: row.job_id,
        type: row.job_type,
        priority: row.priority,
        status: row.status,
        data: JSON.parse(row.data),
        attempts: row.attempts,
        maxRetries: row.max_retries,
        createdAt: row.created_at.getTime(),
        scheduledFor: row.scheduled_for.getTime(),
        metadata: {},
        logs: [],
        progress: 0
      }));
    }
    
    return [];
  }

  async removePersistedJob(jobId) {
    if (this.database) {
      await this.database.execute(`
        DELETE FROM ${this.database.getTableName('queue')}
        WHERE job_id = $1
      `, [jobId]);
    }
  }

  // Helper methods
  
  processNextJob() {
    // Trigger worker processing
    setImmediate(() => {
      // Workers will pick up in their loops
    });
  }

  scheduleDelayedJob(job) {
    const delay = job.scheduledFor - Date.now();
    
    setTimeout(() => {
      this.stats.delayedJobs--;
      this.processNextJob();
    }, Math.max(0, delay));
  }

  updatePerformanceMetrics(job) {
    this.performanceMonitor.jobsProcessed++;
    this.performanceMonitor.totalProcessingTime += job.processingTime;
    
    // Keep last 100 processing times
    this.stats.processingTime.push(job.processingTime);
    if (this.stats.processingTime.length > 100) {
      this.stats.processingTime.shift();
    }
  }

  executeWithTimeout(promise, timeout) {
    return Promise.race([
      promise,
      new Promise((_, reject) => {
        setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout);
      })
    ]);
  }

  validateJob(job) {
    if (!job.type) throw new Error('Job type is required');
    if (!job.data) throw new Error('Job data is required');
    if (job.priority < -1000 || job.priority > 1000) {
      throw new Error('Priority must be between -1000 and 1000');
    }
  }

  async waitForActiveJobs() {
    const checkInterval = 100;
    const maxWait = 30000; // 30 seconds
    let waited = 0;
    
    while (this.stats.activeJobs > 0 && waited < maxWait) {
      await this.sleep(checkInterval);
      waited += checkInterval;
    }
    
    if (this.stats.activeJobs > 0) {
      console.warn(`⚠️ ${this.stats.activeJobs} jobs still active after timeout`);
    }
  }

  startCleanupTimer() {
    this.cleanupTimer = setInterval(() => {
      this.cleanup();
    }, this.config.cleanupInterval);
  }

  generateJobId() {
    return `JOB-${crypto.randomUUID().substring(0, 12).toUpperCase()}`;
  }

  sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  async processItem(item, processorName) {
    const processor = this.processors.get(processorName);
    if (!processor) {
      throw new Error(`Processor not found: ${processorName}`);
    }
    
    return await processor.handler({ data: item });
  }

  /**
   * Graceful shutdown
   */
  async shutdown() {
    console.log('🔄 Graceful shutdown initiated...');
    
    await this.stop();
    
    if (this.cleanupTimer) {
      clearInterval(this.cleanupTimer);
    }
    
    if (this.redis) {
      await this.redis.quit();
    }
    
    console.log('👋 Queue Manager shutdown complete');
    this.emit('queue:shutdown');
  }
}

module.exports = { QueueManager };