claude-flow
Version:
Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration
703 lines (600 loc) • 18.9 kB
text/typescript
/**
* Worker Queue Service
* Redis-based task queue for distributed headless worker execution.
*
* ADR-020: Headless Worker Integration Architecture - Phase 4
* - Priority-based task scheduling
* - Result persistence and retrieval
* - Distributed locking for task assignment
* - Dead letter queue for failed tasks
* - Metrics and monitoring
*
* Key Features:
* - FIFO with priority levels (critical, high, normal, low)
* - Automatic retry with exponential backoff
* - Task timeout detection
* - Result caching with TTL
* - Worker heartbeat monitoring
*/
import { EventEmitter } from 'events';
import { randomUUID } from 'crypto';
import type { HeadlessWorkerType, HeadlessExecutionResult, WorkerPriority } from './headless-worker-executor.js';
// ============================================
// Type Definitions
// ============================================
/**
* Task status
*/
export type TaskStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'timeout' | 'cancelled';
/**
* Queue task
*/
export interface QueueTask {
id: string;
workerType: HeadlessWorkerType;
priority: WorkerPriority;
payload: {
prompt?: string;
contextPatterns?: string[];
sandbox?: string;
model?: string;
timeoutMs?: number;
};
status: TaskStatus;
createdAt: Date;
startedAt?: Date;
completedAt?: Date;
workerId?: string;
retryCount: number;
maxRetries: number;
result?: HeadlessExecutionResult;
error?: string;
}
/**
* Queue configuration
*/
export interface WorkerQueueConfig {
/** Redis connection URL */
redisUrl: string;
/** Queue name prefix */
queuePrefix: string;
/** Default task timeout in ms */
defaultTimeoutMs: number;
/** Maximum retries for failed tasks */
maxRetries: number;
/** Task result TTL in seconds */
resultTtlSeconds: number;
/** Worker heartbeat interval in ms */
heartbeatIntervalMs: number;
/** Dead letter queue enabled */
deadLetterEnabled: boolean;
/** Visibility timeout in ms (task processing lock) */
visibilityTimeoutMs: number;
}
/**
* Queue statistics
*/
export interface QueueStats {
pending: number;
processing: number;
completed: number;
failed: number;
deadLetter: number;
byPriority: Record<WorkerPriority, number>;
byWorkerType: Partial<Record<HeadlessWorkerType, number>>;
averageWaitTimeMs: number;
averageProcessingTimeMs: number;
}
/**
* Worker registration info
*/
export interface WorkerRegistration {
workerId: string;
workerTypes: HeadlessWorkerType[];
maxConcurrent: number;
currentTasks: number;
lastHeartbeat: Date;
registeredAt: Date;
hostname?: string;
containerId?: string;
}
// ============================================
// Constants
// ============================================
const DEFAULT_CONFIG: WorkerQueueConfig = {
redisUrl: 'redis://localhost:6379',
queuePrefix: 'claude-flow:queue',
defaultTimeoutMs: 300000, // 5 minutes
maxRetries: 3,
resultTtlSeconds: 86400, // 24 hours
heartbeatIntervalMs: 30000, // 30 seconds
deadLetterEnabled: true,
visibilityTimeoutMs: 60000, // 1 minute
};
const PRIORITY_SCORES: Record<WorkerPriority, number> = {
critical: 4,
high: 3,
normal: 2,
low: 1,
};
// ============================================
// In-Memory Redis Simulation (for non-Redis environments)
// ============================================
/**
* Simple in-memory queue implementation for environments without Redis
* Production should use actual Redis connection
*/
class InMemoryStore {
private tasks: Map<string, QueueTask> = new Map();
private queues: Map<string, string[]> = new Map();
private workers: Map<string, WorkerRegistration> = new Map();
private results: Map<string, { result: HeadlessExecutionResult; expiresAt: number }> = new Map();
private cleanupTimer?: NodeJS.Timeout;
/**
* Start cleanup timer (called after initialization)
*/
startCleanup(): void {
if (this.cleanupTimer) return;
this.cleanupTimer = setInterval(() => this.cleanupExpired(), 60000);
this.cleanupTimer.unref();
}
/**
* Stop cleanup timer
*/
stopCleanup(): void {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = undefined;
}
}
// Task operations
setTask(id: string, task: QueueTask): void {
this.tasks.set(id, task);
}
getTask(id: string): QueueTask | undefined {
return this.tasks.get(id);
}
deleteTask(id: string): void {
this.tasks.delete(id);
}
// Queue operations
pushToQueue(queue: string, taskId: string, priority: number): void {
const queueTasks = this.queues.get(queue) || [];
// Insert based on priority (higher priority = earlier in queue)
let insertIndex = queueTasks.length;
for (let i = 0; i < queueTasks.length; i++) {
const task = this.tasks.get(queueTasks[i]);
if (task && PRIORITY_SCORES[task.priority] < priority) {
insertIndex = i;
break;
}
}
queueTasks.splice(insertIndex, 0, taskId);
this.queues.set(queue, queueTasks);
}
popFromQueue(queue: string): string | null {
const queueTasks = this.queues.get(queue) || [];
if (queueTasks.length === 0) return null;
return queueTasks.shift() || null;
}
getQueueLength(queue: string): number {
return (this.queues.get(queue) || []).length;
}
// Worker operations
setWorker(workerId: string, registration: WorkerRegistration): void {
this.workers.set(workerId, registration);
}
getWorker(workerId: string): WorkerRegistration | undefined {
return this.workers.get(workerId);
}
deleteWorker(workerId: string): void {
this.workers.delete(workerId);
}
getAllWorkers(): WorkerRegistration[] {
return Array.from(this.workers.values());
}
// Result operations
setResult(taskId: string, result: HeadlessExecutionResult, ttlSeconds: number): void {
this.results.set(taskId, {
result,
expiresAt: Date.now() + ttlSeconds * 1000,
});
}
getResult(taskId: string): HeadlessExecutionResult | undefined {
const entry = this.results.get(taskId);
if (!entry) return undefined;
if (Date.now() > entry.expiresAt) {
this.results.delete(taskId);
return undefined;
}
return entry.result;
}
// Stats
getStats(): { tasks: number; workers: number; results: number } {
return {
tasks: this.tasks.size,
workers: this.workers.size,
results: this.results.size,
};
}
// Cleanup
private cleanupExpired(): void {
const now = Date.now();
for (const [id, entry] of this.results) {
if (now > entry.expiresAt) {
this.results.delete(id);
}
}
}
}
// ============================================
// WorkerQueue Class
// ============================================
/**
* WorkerQueue - Redis-based task queue for distributed worker execution
*/
export class WorkerQueue extends EventEmitter {
private config: WorkerQueueConfig;
private store: InMemoryStore;
private workerId: string;
private heartbeatTimer?: NodeJS.Timeout;
private processingTasks: Set<string> = new Set();
private isShuttingDown = false;
private maxConcurrent = 1;
private initialized = false;
constructor(config?: Partial<WorkerQueueConfig>) {
super();
this.config = { ...DEFAULT_CONFIG, ...config };
this.store = new InMemoryStore();
this.workerId = `worker-${randomUUID().slice(0, 8)}`;
}
/**
* Initialize the queue (starts cleanup timers)
*/
async initialize(): Promise<void> {
if (this.initialized) return;
this.store.startCleanup();
this.initialized = true;
this.emit('initialized', { workerId: this.workerId });
}
// ============================================
// Public API - Task Management
// ============================================
/**
* Enqueue a new task
*/
async enqueue(
workerType: HeadlessWorkerType,
payload: QueueTask['payload'] = {},
options?: {
priority?: WorkerPriority;
maxRetries?: number;
timeoutMs?: number;
}
): Promise<string> {
// Initialize if needed
if (!this.initialized) {
await this.initialize();
}
// Validate worker type
if (!workerType || typeof workerType !== 'string') {
throw new Error('Invalid worker type');
}
// Validate priority
const priority = options?.priority || 'normal';
if (!['critical', 'high', 'normal', 'low'].includes(priority)) {
throw new Error(`Invalid priority: ${priority}`);
}
const taskId = `task-${Date.now()}-${randomUUID().slice(0, 8)}`;
const task: QueueTask = {
id: taskId,
workerType,
priority,
payload: {
...payload,
timeoutMs: options?.timeoutMs || this.config.defaultTimeoutMs,
},
status: 'pending',
createdAt: new Date(),
retryCount: 0,
maxRetries: options?.maxRetries ?? this.config.maxRetries,
};
// Store task
this.store.setTask(taskId, task);
// Add to priority queue
const queueName = this.getQueueName(workerType);
this.store.pushToQueue(queueName, taskId, PRIORITY_SCORES[priority]);
this.emit('taskEnqueued', { taskId, workerType, priority });
return taskId;
}
/**
* Dequeue a task for processing
*/
async dequeue(workerTypes: HeadlessWorkerType[]): Promise<QueueTask | null> {
if (this.isShuttingDown) return null;
// Check queues in priority order
for (const workerType of workerTypes) {
const queueName = this.getQueueName(workerType);
const taskId = this.store.popFromQueue(queueName);
if (taskId) {
const task = this.store.getTask(taskId);
if (task && task.status === 'pending') {
task.status = 'processing';
task.startedAt = new Date();
task.workerId = this.workerId;
this.store.setTask(taskId, task);
this.processingTasks.add(taskId);
this.emit('taskDequeued', { taskId, workerType });
return task;
}
}
}
return null;
}
/**
* Complete a task with result
*/
async complete(taskId: string, result: HeadlessExecutionResult): Promise<void> {
const task = this.store.getTask(taskId);
if (!task) {
this.emit('warning', { message: `Task ${taskId} not found for completion` });
return;
}
task.status = 'completed';
task.completedAt = new Date();
task.result = result;
this.store.setTask(taskId, task);
// Store result with TTL
this.store.setResult(taskId, result, this.config.resultTtlSeconds);
this.processingTasks.delete(taskId);
this.emit('taskCompleted', { taskId, result, duration: task.completedAt.getTime() - (task.startedAt?.getTime() || 0) });
}
/**
* Fail a task with error
*/
async fail(taskId: string, error: string, retryable = true): Promise<void> {
const task = this.store.getTask(taskId);
if (!task) {
this.emit('warning', { message: `Task ${taskId} not found for failure` });
return;
}
this.processingTasks.delete(taskId);
// Check if we should retry
if (retryable && task.retryCount < task.maxRetries) {
task.retryCount++;
task.status = 'pending';
task.startedAt = undefined;
task.workerId = undefined;
task.error = error;
this.store.setTask(taskId, task);
// Re-queue with delay (exponential backoff)
const delay = Math.min(30000, 1000 * Math.pow(2, task.retryCount));
setTimeout(() => {
const queueName = this.getQueueName(task.workerType);
this.store.pushToQueue(queueName, taskId, PRIORITY_SCORES[task.priority]);
}, delay);
this.emit('taskRetrying', { taskId, retryCount: task.retryCount, delay });
} else {
// Move to failed/dead letter
task.status = 'failed';
task.completedAt = new Date();
task.error = error;
this.store.setTask(taskId, task);
if (this.config.deadLetterEnabled) {
const dlqName = `${this.config.queuePrefix}:dlq`;
this.store.pushToQueue(dlqName, taskId, 0);
}
this.emit('taskFailed', { taskId, error, retryCount: task.retryCount });
}
}
/**
* Get task status
*/
async getTask(taskId: string): Promise<QueueTask | null> {
return this.store.getTask(taskId) || null;
}
/**
* Get task result
*/
async getResult(taskId: string): Promise<HeadlessExecutionResult | null> {
return this.store.getResult(taskId) || null;
}
/**
* Cancel a pending task
*/
async cancel(taskId: string): Promise<boolean> {
const task = this.store.getTask(taskId);
if (!task || task.status !== 'pending') {
return false;
}
task.status = 'cancelled';
task.completedAt = new Date();
this.store.setTask(taskId, task);
this.emit('taskCancelled', { taskId });
return true;
}
// ============================================
// Public API - Worker Management
// ============================================
/**
* Register this instance as a worker
*/
async registerWorker(
workerTypes: HeadlessWorkerType[],
options?: { maxConcurrent?: number; hostname?: string; containerId?: string }
): Promise<string> {
// Initialize if needed
if (!this.initialized) {
await this.initialize();
}
// Validate worker types
if (!Array.isArray(workerTypes) || workerTypes.length === 0) {
throw new Error('Worker types must be a non-empty array');
}
this.maxConcurrent = options?.maxConcurrent || 1;
const registration: WorkerRegistration = {
workerId: this.workerId,
workerTypes,
maxConcurrent: this.maxConcurrent,
currentTasks: 0,
lastHeartbeat: new Date(),
registeredAt: new Date(),
hostname: options?.hostname,
containerId: options?.containerId,
};
this.store.setWorker(this.workerId, registration);
// Start heartbeat
this.startHeartbeat();
this.emit('workerRegistered', { workerId: this.workerId, workerTypes });
return this.workerId;
}
/**
* Unregister this worker
*/
async unregisterWorker(): Promise<void> {
this.stopHeartbeat();
this.store.deleteWorker(this.workerId);
this.emit('workerUnregistered', { workerId: this.workerId });
}
/**
* Get all registered workers
*/
async getWorkers(): Promise<WorkerRegistration[]> {
return this.store.getAllWorkers();
}
// ============================================
// Public API - Statistics
// ============================================
/**
* Get queue statistics
*/
async getStats(): Promise<QueueStats> {
const storeStats = this.store.getStats();
// This is a simplified implementation
// Full implementation would aggregate across all queues
const stats: QueueStats = {
pending: 0,
processing: this.processingTasks.size,
completed: 0,
failed: 0,
deadLetter: 0,
byPriority: { critical: 0, high: 0, normal: 0, low: 0 },
byWorkerType: {},
averageWaitTimeMs: 0,
averageProcessingTimeMs: 0,
};
return stats;
}
// ============================================
// Public API - Lifecycle
// ============================================
/**
* Start processing tasks
*/
async start(
workerTypes: HeadlessWorkerType[],
handler: (task: QueueTask) => Promise<HeadlessExecutionResult>,
options?: { maxConcurrent?: number }
): Promise<void> {
await this.registerWorker(workerTypes, { maxConcurrent: options?.maxConcurrent });
const processLoop = async () => {
while (!this.isShuttingDown) {
try {
// Respect concurrency limit
if (this.processingTasks.size >= this.maxConcurrent) {
await new Promise(resolve => setTimeout(resolve, 100));
continue;
}
const task = await this.dequeue(workerTypes);
if (task) {
// Process task without blocking the loop (allows concurrency)
this.processTask(task, handler).catch(error => {
this.emit('error', { taskId: task.id, error: String(error) });
});
} else {
// No task available, wait before polling again
await new Promise(resolve => setTimeout(resolve, 1000));
}
} catch (error) {
this.emit('error', { error: error instanceof Error ? error.message : String(error) });
// Wait before retrying to avoid tight error loop
await new Promise(resolve => setTimeout(resolve, 5000));
}
}
};
// Start processing
processLoop().catch(error => {
this.emit('error', { error: error instanceof Error ? error.message : String(error) });
});
}
/**
* Process a single task
*/
private async processTask(
task: QueueTask,
handler: (task: QueueTask) => Promise<HeadlessExecutionResult>
): Promise<void> {
try {
const result = await handler(task);
await this.complete(task.id, result);
} catch (error) {
await this.fail(task.id, error instanceof Error ? error.message : String(error));
}
}
/**
* Shutdown the queue gracefully
*/
async shutdown(): Promise<void> {
if (this.isShuttingDown) return;
this.isShuttingDown = true;
// Wait for processing tasks to complete (with timeout)
const timeout = 30000;
const start = Date.now();
while (this.processingTasks.size > 0 && Date.now() - start < timeout) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Force fail remaining tasks
for (const taskId of this.processingTasks) {
await this.fail(taskId, 'Worker shutdown', false);
}
// Stop store cleanup
this.store.stopCleanup();
await this.unregisterWorker();
this.initialized = false;
this.emit('shutdown', {});
}
// ============================================
// Private Methods
// ============================================
/**
* Get queue name for worker type
*/
private getQueueName(workerType: HeadlessWorkerType): string {
return `${this.config.queuePrefix}:${workerType}`;
}
/**
* Start heartbeat timer
*/
private startHeartbeat(): void {
this.heartbeatTimer = setInterval(() => {
const registration = this.store.getWorker(this.workerId);
if (registration) {
registration.lastHeartbeat = new Date();
registration.currentTasks = this.processingTasks.size;
this.store.setWorker(this.workerId, registration);
}
}, this.config.heartbeatIntervalMs);
this.heartbeatTimer.unref();
}
/**
* Stop heartbeat timer
*/
private stopHeartbeat(): void {
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = undefined;
}
}
}
// Export default
export default WorkerQueue;