@clipwhisperer/common/src/infrastructure/queue.ts

ClipWhisperer Common - Shared library providing core utilities, database schemas, authentication, bucket management, and common functionality across all ClipWhisperer microservices

/**
 * Queue Infrastructure
 * Centralized BullMQ queue and worker management
 */

import { Queue, Worker, Job } from "bullmq";
import { REDIS_CONNECTION_CONFIG, DEFAULT_QUEUE_OPTIONS, QUEUE_NAMES } from "../constants/redis";
import { PerformanceTracker } from "./performance";

export type QueueJobInput = Record<string, any>;

export type QueueJobOutput = Record<string, any>;

export interface QueueConfig {
  name: string;
  connection?: {
    host: string;
    port: number;
  };
  defaultJobOptions?: {
    removeOnComplete?: boolean;
    removeOnFail?: boolean;
    attempts?: number;
    backoff?: {
      type: string;
      delay: number;
    };
  };
}

/**
 * Queue Factory - Creates standardized BullMQ queues
 */
export class QueueFactory {
  /**
   * Create a queue with standard configuration
   */
  static createQueue<TInput extends QueueJobInput, TOutput extends QueueJobOutput>(
    queueName: string,
    config?: Partial<QueueConfig>
  ): Queue<TInput, TOutput> {
    return new Queue<TInput, TOutput>(queueName, {
      connection: config?.connection || REDIS_CONNECTION_CONFIG,
      defaultJobOptions: {
        ...DEFAULT_QUEUE_OPTIONS,
        ...config?.defaultJobOptions,
      },
    });
  }

  /**
   * Create a worker with standard configuration
   */
  static createWorker<TInput extends QueueJobInput, TOutput extends QueueJobOutput>(
    queueName: string,
    processor: (job: Job<TInput, TOutput>) => Promise<TOutput>,
    config?: Partial<QueueConfig>
  ): Worker<TInput, TOutput> {
    return new Worker<TInput, TOutput>(
      queueName,
      processor,
      {
        connection: config?.connection || REDIS_CONNECTION_CONFIG,
        ...config,
      }
    );
  }
}

/**
 * Base Queue Manager - Provides common queue operations
 */
export abstract class BaseQueueManager<TInput extends QueueJobInput, TOutput extends QueueJobOutput> {
  protected queue: Queue<TInput, TOutput>;
  protected worker?: Worker<TInput, TOutput>;
  protected performanceTracker: PerformanceTracker;

  constructor(
    protected queueName: string,
    protected config?: Partial<QueueConfig>
  ) {
    this.queue = QueueFactory.createQueue<TInput, TOutput>(queueName, config);
    this.performanceTracker = new PerformanceTracker();
  }

  /**
 * Add a job to the queue
 */
  async addJob(
    jobName: string,
    data: TInput,
    options?: {
      removeOnComplete?: boolean;
      removeOnFail?: boolean;
      delay?: number;
      attempts?: number;
    }
  ): Promise<{ jobId: string }> {
    const startTime = Date.now();

    try {
      const job = await (this.queue as any).add(jobName, data, {
        ...DEFAULT_QUEUE_OPTIONS,
        ...options,
      });

      this.performanceTracker.trackSuccess(Date.now() - startTime);
      return { jobId: job.id || `job_${Date.now()}` };
    } catch (error) {
      this.performanceTracker.trackError(Date.now() - startTime);
      throw error;
    }
  }

  /**
   * Start the worker
   */
  startWorker(processor: (job: Job<TInput, TOutput>) => Promise<TOutput>): void {
    this.worker = QueueFactory.createWorker<TInput, TOutput>(
      this.queueName,
      async (job) => {
        return await this.performanceTracker.trackRequest(async () => {
          return await processor(job);
        });
      },
      this.config
    );

    this.worker.on("completed", (job) => {
      console.log(`✅ Job ${job.id} completed successfully`);
    });

    this.worker.on("failed", (job, err) => {
      console.error(`❌ Job ${job?.id} failed:`, err);
    });
  }

  /**
   * Get queue statistics
   */
  async getQueueStats(): Promise<{
    waiting: number;
    active: number;
    completed: number;
    failed: number;
  }> {
    return {
      waiting: await this.queue.getWaiting().then(jobs => jobs.length),
      active: await this.queue.getActive().then(jobs => jobs.length),
      completed: await this.queue.getCompleted().then(jobs => jobs.length),
      failed: await this.queue.getFailed().then(jobs => jobs.length),
    };
  }

  /**
   * Get enhanced performance metrics including queue size
   */
  async getPerformanceMetrics() {
    const baseMetrics = this.performanceTracker.getMetrics();
    const queueStats = await this.getQueueStats();

    return {
      ...baseMetrics,
      queueSize: queueStats.waiting + queueStats.active,
      queueStats,
    };
  }

  /**
   * Close queue and worker connections
   */
  async close(): Promise<void> {
    await this.queue.close();
    if (this.worker) {
      await this.worker.close();
    }
  }
}