claude-flow-novice
Version:
Claude Flow Novice - Advanced orchestration platform for multi-agent AI workflows with CFN Loop architecture Includes Local RuVector Accelerator and all CFN skills for complete functionality.
654 lines (653 loc) • 25.1 kB
JavaScript
/**
* Enhanced Progress Tracker with Granular Progress Updates and Redis Messaging
*
* Provides detailed progress tracking for agent tasks with real-time Redis pub/sub messaging.
* Supports granular progress updates, confidence scoring, and comprehensive visibility
* into agent execution states.
*/ import { EventEmitter } from 'events';
import { createClient } from 'redis';
import { Logger } from '../core/logger.js';
// ===== REDIS CHANNELS AND KEYS =====
export const REDIS_CHANNELS = {
PROGRESS_UPDATES: 'progress:updates',
AGENT_VISIBILITY: 'agent:visibility',
SWARM_OVERVIEW: 'swarm:overview',
TASK_EVENTS: 'task:events',
COORDINATION_SIGNALS: 'coordination:signals'
};
export const REDIS_KEYS = {
AGENT_PROGRESS: 'agent:progress:',
TASK_PROGRESS: 'task:progress:',
SWARM_OVERVIEW: 'swarm:overview:',
AGENT_VISIBILITY: 'agent:visibility:',
PROGRESS_HISTORY: 'progress:history:'
};
// ===== ENHANCED PROGRESS TRACKER CLASS =====
export class EnhancedProgressTracker extends EventEmitter {
redis;
subscriber;
logger;
taskProgress = new Map();
agentVisibility = new Map();
swarmOverviews = new Map();
subscriptions = new Map();
hmacSecret;
constructor(redisUrl = process.env.CFN_REDIS_URL || process.env.REDIS_URL || `redis://${process.env.CFN_REDIS_HOST || 'cfn-redis'}:${process.env.CFN_REDIS_PORT || 6379}`, loggerConfig, hmacSecret = process.env.HMAC_SECRET || 'default-secret'){
super();
this.hmacSecret = hmacSecret;
// Initialize logger
const config = loggerConfig || {
level: process.env.CLAUDE_FLOW_ENV === 'test' ? 'error' : 'info',
format: 'json',
destination: 'console'
};
this.logger = new Logger(config, {
component: 'EnhancedProgressTracker'
});
// Initialize Redis clients
this.redis = createClient({
url: redisUrl
});
this.subscriber = createClient({
url: redisUrl
});
this.setupRedisClients();
}
/**
* Initialize Redis connections and subscriptions
*/ async initialize() {
try {
await Promise.all([
this.redis.connect(),
this.subscriber.connect()
]);
// Set up default subscriptions
await this.setupDefaultSubscriptions();
this.logger.info('Enhanced Progress Tracker initialized', {
redisConnected: true,
subscriptionsEnabled: true
});
this.emit('initialized');
} catch (error) {
this.logger.error('Failed to initialize Enhanced Progress Tracker', {
error: error instanceof Error ? error.message : String(error)
});
throw error;
}
}
/**
* Create a new task progress tracker
*/ async createTaskProgress(taskId, agentId, swarmId, taskType, taskDescription, steps) {
const progressSteps = steps.map((step, index)=>({
...step,
id: `step-${index + 1}`,
status: 'pending'
}));
const taskProgress = {
taskId,
agentId,
swarmId,
taskType,
taskDescription,
overallStatus: 'pending',
progressPercentage: 0,
steps: progressSteps,
startTime: Date.now(),
confidence: 0.5,
metadata: {
filesProcessed: [],
deliverables: [],
dependencies: [],
blockers: [],
resources: {}
}
};
// Store in memory
this.taskProgress.set(taskId, taskProgress);
// Store in Redis with TTL
await this.redis.setEx(`${REDIS_KEYS.TASK_PROGRESS}${taskId}`, 86400, JSON.stringify(taskProgress));
// Publish creation event
await this.publishProgressUpdate({
type: 'progress_update',
agentId,
swarmId,
taskId,
timestamp: Date.now(),
data: taskProgress
});
this.logger.info('Task progress created', {
taskId,
agentId,
swarmId,
taskType,
stepCount: steps.length
});
}
/**
* Update task progress with granular step information
*/ async updateTaskProgress(taskId, updates) {
const existing = this.taskProgress.get(taskId);
if (!existing) {
throw new Error(`Task progress not found: ${taskId}`);
}
// Update task progress
const updated = {
...existing,
...updates,
metadata: {
...existing.metadata,
...updates.metadata
},
reasoning: {
...existing.reasoning,
...updates.reasoning
}
};
// Update specific step if provided
if (updates.stepId) {
const step = updated.steps.find((s)=>s.id === updates.stepId);
if (step) {
if (updates.status && this.isStepStatus(updates.status)) {
step.status = updates.status;
}
if (updates.status === 'in_progress' && !step.startTime) {
step.startTime = Date.now();
}
if (updates.status === 'completed' || updates.status === 'failed') {
step.endTime = Date.now();
step.duration = step.endTime - (step.startTime || step.endTime);
}
if (updates.confidence) {
step.confidence = updates.confidence;
}
if (updates.error) {
step.error = updates.error;
}
}
}
// Recalculate overall progress
updated.progressPercentage = this.calculateOverallProgress(updated.steps);
// Update in memory
this.taskProgress.set(taskId, updated);
// Update in Redis
await this.redis.setEx(`${REDIS_KEYS.TASK_PROGRESS}${taskId}`, 86400, JSON.stringify(updated));
// Publish update
await this.publishProgressUpdate({
type: 'progress_update',
agentId: updated.agentId,
swarmId: updated.swarmId,
taskId,
timestamp: Date.now(),
data: updated
});
// Check for task completion
if (updated.progressPercentage >= 100 && updated.overallStatus !== 'completed') {
await this.completeTask(taskId);
}
this.logger.debug('Task progress updated', {
taskId,
progressPercentage: updated.progressPercentage,
stepId: updates.stepId,
status: updated.overallStatus
});
}
/**
* Add sub-steps to an existing step
*/ async addSubSteps(taskId, parentStepId, subSteps) {
const task = this.taskProgress.get(taskId);
if (!task) {
throw new Error(`Task progress not found: ${taskId}`);
}
const parentStep = task.steps.find((s)=>s.id === parentStepId);
if (!parentStep) {
throw new Error(`Parent step not found: ${parentStepId}`);
}
const newSubSteps = subSteps.map((step, index)=>({
...step,
id: `${parentStepId}-sub-${index + 1}`,
status: 'pending'
}));
parentStep.subSteps = [
...parentStep.subSteps || [],
...newSubSteps
];
// Update in memory and Redis
this.taskProgress.set(taskId, task);
await this.redis.setEx(`${REDIS_KEYS.TASK_PROGRESS}${taskId}`, 86400, JSON.stringify(task));
this.logger.debug('Sub-steps added', {
taskId,
parentStepId,
subStepCount: subSteps.length
});
}
/**
* Mark task as completed
*/ async completeTask(taskId, deliverables) {
const task = this.taskProgress.get(taskId);
if (!task) {
throw new Error(`Task progress not found: ${taskId}`);
}
task.overallStatus = 'completed';
task.endTime = Date.now();
task.progressPercentage = 100;
task.confidence = Math.max(task.confidence, 0.8);
if (deliverables) {
task.metadata.deliverables = [
...task.metadata.deliverables || [],
...deliverables
];
}
// Update in memory and Redis
this.taskProgress.set(taskId, task);
await this.redis.setEx(`${REDIS_KEYS.TASK_PROGRESS}${taskId}`, 86400, JSON.stringify(task));
// Update agent visibility
await this.updateAgentVisibility(task.agentId, {
status: 'completed',
performance: {
tasksCompleted: (this.agentVisibility.get(task.agentId)?.performance.tasksCompleted || 0) + 1,
averageTaskDuration: this.calculateAverageTaskDuration(task.agentId),
successRate: this.calculateSuccessRate(task.agentId),
currentStreak: (this.agentVisibility.get(task.agentId)?.performance.currentStreak || 0) + 1
}
});
// Publish completion event
await this.publishProgressUpdate({
type: 'task_complete',
agentId: task.agentId,
swarmId: task.swarmId,
taskId,
timestamp: Date.now(),
data: task
});
this.logger.info('Task completed', {
taskId,
agentId: task.agentId,
duration: task.endTime - task.startTime,
deliverables: task.metadata.deliverables?.length || 0
});
}
/**
* Mark task as failed
*/ async failTask(taskId, error, details) {
const task = this.taskProgress.get(taskId);
if (!task) {
throw new Error(`Task progress not found: ${taskId}`);
}
task.overallStatus = 'failed';
task.endTime = Date.now();
task.confidence = Math.min(task.confidence, 0.2);
// Add error to current step or task metadata
if (task.currentStep) {
const currentStep = task.steps.find((s)=>s.id === task.currentStep);
if (currentStep) {
currentStep.status = 'failed';
currentStep.error = error;
currentStep.endTime = Date.now();
}
}
task.metadata.blockers = [
...task.metadata.blockers || [],
error
];
// Update in memory and Redis
this.taskProgress.set(taskId, task);
await this.redis.setEx(`${REDIS_KEYS.TASK_PROGRESS}${taskId}`, 86400, JSON.stringify(task));
// Update agent visibility
await this.updateAgentVisibility(task.agentId, {
status: 'error',
performance: {
tasksCompleted: this.agentVisibility.get(task.agentId)?.performance.tasksCompleted || 0,
averageTaskDuration: this.calculateAverageTaskDuration(task.agentId),
successRate: this.calculateSuccessRate(task.agentId),
currentStreak: 0
}
});
// Publish failure event
await this.publishProgressUpdate({
type: 'task_failed',
agentId: task.agentId,
swarmId: task.swarmId,
taskId,
timestamp: Date.now(),
data: {
...task,
error,
details: details ?? {}
}
});
this.logger.error('Task failed', {
taskId,
agentId: task.agentId,
error,
duration: task.endTime - task.startTime
});
}
/**
* Update agent visibility information
*/ async updateAgentVisibility(agentId, updates) {
const existing = this.agentVisibility.get(agentId) || this.createDefaultAgentVisibility(agentId);
const updated = {
...existing,
...updates,
recentActivity: [
{
timestamp: Date.now(),
action: 'status_update',
details: updates.status ? `Status updated to ${updates.status}` : `Status unchanged: ${existing.status}`
},
...existing.recentActivity?.slice(0, 49) || [] // Keep last 50 activities
]
};
this.agentVisibility.set(agentId, updated);
// Store in Redis (null check)
if (this.redis) {
await this.redis.setEx(`${REDIS_KEYS.AGENT_VISIBILITY}${agentId}`, 3600, JSON.stringify(updated));
// Publish visibility update (null check)
await this.redis.publish(REDIS_CHANNELS.AGENT_VISIBILITY, JSON.stringify({
type: 'visibility_update',
agentId,
timestamp: Date.now(),
data: updated
}));
}
this.emit('agent-visibility-updated', {
agentId,
visibility: updated
});
}
/**
* Get comprehensive task progress
*/ async getTaskProgress(taskId) {
// Try memory first
let progress = this.taskProgress.get(taskId);
// Fallback to Redis
if (!progress) {
const stored = await this.redis.get(`${REDIS_KEYS.TASK_PROGRESS}${taskId}`);
if (stored) {
progress = JSON.parse(stored);
this.taskProgress.set(taskId, progress);
}
}
return progress || null;
}
/**
* Get agent visibility information
*/ async getAgentVisibility(agentId) {
// Try memory first
let visibility = this.agentVisibility.get(agentId);
// Fallback to Redis
if (!visibility) {
const stored = await this.redis.get(`${REDIS_KEYS.AGENT_VISIBILITY}${agentId}`);
if (stored) {
visibility = JSON.parse(stored);
this.agentVisibility.set(agentId, visibility);
}
}
return visibility || null;
}
/**
* Get swarm progress overview
*/ async getSwarmOverview(swarmId) {
// Try memory first
let overview = this.swarmOverviews.get(swarmId);
// If not in memory or stale, recalculate
if (!overview || Date.now() - overview.lastUpdated > 30000) {
overview = await this.calculateSwarmOverview(swarmId);
this.swarmOverviews.set(swarmId, overview);
}
return overview;
}
/**
* Get active tasks for an agent
*/ async getActiveTasks(agentId) {
const tasks = [];
// Check memory
for (const [taskId, task] of this.taskProgress){
if (task.agentId === agentId && task.overallStatus === 'in_progress') {
tasks.push(task);
}
}
// Check Redis for any additional tasks
const pattern = `${REDIS_KEYS.TASK_PROGRESS}*`;
const keys = await this.redis.keys(pattern);
for (const key of keys){
if (!this.taskProgress.has(key.replace(REDIS_KEYS.TASK_PROGRESS, ''))) {
const stored = await this.redis.get(key);
if (stored) {
const task = JSON.parse(stored);
if (task.agentId === agentId && task.overallStatus === 'in_progress') {
tasks.push(task);
}
}
}
}
return tasks;
}
/**
* Subscribe to progress updates for specific agents or swarms
*/ async subscribeToProgress(filter, callback) {
const subscriptionKey = JSON.stringify(filter);
if (!this.subscriptions.has(subscriptionKey)) {
this.subscriptions.set(subscriptionKey, new Set());
}
this.subscriptions.get(subscriptionKey).add(callback);
// Subscribe to Redis channel if not already subscribed
await this.subscriber.subscribe(REDIS_CHANNELS.PROGRESS_UPDATES, (message)=>{
try {
const update = JSON.parse(message);
// Apply filter
if (filter.agentIds && !filter.agentIds.includes(update.agentId)) return;
if (filter.swarmIds && !filter.swarmIds.includes(update.swarmId)) return;
// Get task details for task type filtering
if (filter.taskTypes) {
const task = this.taskProgress.get(update.taskId);
if (!task || !filter.taskTypes.includes(task.taskType)) return;
}
callback(update);
} catch (error) {
this.logger.error('Error processing progress update', {
error: error instanceof Error ? error.message : String(error),
message
});
}
});
this.logger.debug('Subscribed to progress updates', {
filter
});
}
/**
* Unsubscribe from progress updates
*/ async unsubscribeFromProgress(filter, callback) {
const subscriptionKey = JSON.stringify(filter);
const subscriptions = this.subscriptions.get(subscriptionKey);
if (subscriptions) {
if (callback) {
subscriptions.delete(callback);
} else {
subscriptions.clear();
}
if (subscriptions.size === 0) {
this.subscriptions.delete(subscriptionKey);
}
}
}
/**
* Cleanup resources
*/ async cleanup() {
try {
await Promise.all([
this.redis.quit(),
this.subscriber.quit()
]);
this.taskProgress.clear();
this.agentVisibility.clear();
this.swarmOverviews.clear();
this.subscriptions.clear();
this.logger.info('Enhanced Progress Tracker cleaned up');
} catch (error) {
this.logger.error('Error during cleanup', {
error: error instanceof Error ? error.message : String(error)
});
}
}
// ===== PRIVATE METHODS =====
setupRedisClients() {
this.redis.on('error', (err)=>{
this.logger.error('Redis client error', {
error: err.message
});
});
this.subscriber.on('error', (err)=>{
this.logger.error('Redis subscriber error', {
error: err.message
});
});
}
async setupDefaultSubscriptions() {
// Subscribe to agent visibility updates
await this.subscriber.subscribe(REDIS_CHANNELS.AGENT_VISIBILITY, (message)=>{
try {
const update = JSON.parse(message);
this.emit('agent-visibility-update', update);
} catch (error) {
this.logger.error('Error processing visibility update', {
error
});
}
});
}
async publishProgressUpdate(message) {
// Add HMAC signature for authentication
message.signature = this.generateHmacSignature(message);
await this.redis.publish(REDIS_CHANNELS.PROGRESS_UPDATES, JSON.stringify(message));
}
generateHmacSignature(message) {
const crypto = require('crypto');
const payload = JSON.stringify({
type: message.type,
agentId: message.agentId,
swarmId: message.swarmId,
taskId: message.taskId,
timestamp: message.timestamp
});
return crypto.createHmac('sha256', this.hmacSecret).update(payload).digest('hex');
}
calculateOverallProgress(steps) {
if (steps.length === 0) return 0;
let totalProgress = 0;
let totalWeight = 0;
for (const step of steps){
const stepProgress = this.calculateStepProgress(step);
const weight = 1; // Can be modified to weight steps differently
totalProgress += stepProgress * weight;
totalWeight += weight;
}
return totalWeight > 0 ? Math.round(totalProgress / totalWeight) : 0;
}
calculateStepProgress(step) {
if (step.status === 'completed') return 100;
if (step.status === 'failed') return 0;
if (step.status === 'skipped') return 100;
if (step.status !== 'in_progress') return 0;
// If step has sub-steps, calculate based on sub-steps
if (step.subSteps && step.subSteps.length > 0) {
return this.calculateOverallProgress(step.subSteps);
}
// Default progress for in-progress steps without sub-steps
return 50; // Can be enhanced with time-based estimation
}
isStepStatus(status) {
return [
'pending',
'in_progress',
'completed',
'failed',
'skipped'
].includes(status);
}
createDefaultAgentVisibility(agentId) {
return {
agentId,
agentType: 'unknown',
status: 'idle',
recentActivity: [],
performance: {
tasksCompleted: 0,
averageTaskDuration: 0,
successRate: 1.0,
currentStreak: 0
},
capabilities: [],
availability: {
currentLoad: 0,
maxConcurrentTasks: 1
}
};
}
async calculateSwarmOverview(swarmId) {
const tasks = Array.from(this.taskProgress.values()).filter((t)=>t.swarmId === swarmId);
const agents = Array.from(this.agentVisibility.values()).filter((a)=>this.taskProgress.has(a.agentId) && this.taskProgress.get(a.agentId)?.swarmId === swarmId);
const totalTasks = tasks.length;
const completedTasks = tasks.filter((t)=>t.overallStatus === 'completed').length;
const failedTasks = tasks.filter((t)=>t.overallStatus === 'failed').length;
const activeAgents = agents.filter((a)=>[
'active',
'working'
].includes(a.status)).length;
const overallProgress = totalTasks > 0 ? completedTasks / totalTasks * 100 : 0;
const successRate = totalTasks > 0 ? completedTasks / totalTasks : 1.0;
const healthScore = (successRate * 0.7 + activeAgents / Math.max(1, agents.length) * 0.3) * 100;
// Identify bottlenecks
const bottlenecks = [];
const blockedTasks = tasks.filter((t)=>t.overallStatus === 'blocked');
if (blockedTasks.length > 0) {
bottlenecks.push(`${blockedTasks.length} blocked tasks`);
}
const errorAgents = agents.filter((a)=>a.status === 'error');
if (errorAgents.length > 0) {
bottlenecks.push(`${errorAgents.length} agents in error state`);
}
return {
swarmId,
totalAgents: agents.length,
activeAgents,
totalTasks,
completedTasks,
failedTasks,
overallProgress: Math.round(overallProgress),
estimatedCompletion: this.estimateSwarmCompletion(swarmId),
bottlenecks,
healthScore: Math.round(healthScore),
lastUpdated: Date.now()
};
}
estimateSwarmCompletion(swarmId) {
const tasks = Array.from(this.taskProgress.values()).filter((t)=>t.swarmId === swarmId && t.overallStatus === 'in_progress');
if (tasks.length === 0) return undefined;
const averageTaskTime = tasks.reduce((sum, task)=>{
const elapsed = Date.now() - task.startTime;
const progress = task.progressPercentage / 100;
return sum + (progress > 0 ? elapsed / progress : 0);
}, 0) / tasks.length;
return Date.now() + averageTaskTime;
}
calculateAverageTaskDuration(agentId) {
const completedTasks = Array.from(this.taskProgress.values()).filter((t)=>t.agentId === agentId && t.overallStatus === 'completed' && t.endTime);
if (completedTasks.length === 0) return 0;
const totalDuration = completedTasks.reduce((sum, task)=>sum + (task.endTime - task.startTime), 0);
return totalDuration / completedTasks.length;
}
calculateSuccessRate(agentId) {
const agentTasks = Array.from(this.taskProgress.values()).filter((t)=>t.agentId === agentId);
if (agentTasks.length === 0) return 1.0;
const completedTasks = agentTasks.filter((t)=>t.overallStatus === 'completed').length;
return completedTasks / agentTasks.length;
}
}
// ===== FACTORY FUNCTION =====
export function createEnhancedProgressTracker(redisUrl, loggerConfig, hmacSecret) {
return new EnhancedProgressTracker(redisUrl, loggerConfig, hmacSecret);
}
// ===== EXPORTS =====
export default EnhancedProgressTracker;
//# sourceMappingURL=enhanced-progress-tracker.js.map