agentic-qe/dist/cli/commands/agent/benchmark.js

Agentic Quality Engineering Fleet System - AI-driven quality management platform

"use strict";
/**
 * Agent Benchmark Command
 * Benchmark agent performance with detailed metrics
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.benchmark = void 0;
const Task_1 = require("../../../core/Task");
const Logger_1 = require("../../../utils/Logger");
async function benchmark(options) {
    const logger = Logger_1.Logger.getInstance();
    const iterations = options.iterations || 10;
    try {
        const agent = options.fleetManager.getAgent(options.agentId);
        if (!agent) {
            throw new Error(`Agent ${options.agentId} not found`);
        }
        const latencies = [];
        const completionTimes = [];
        let completedTasks = 0;
        let failedTasks = 0;
        const startTime = Date.now();
        // Run benchmark iterations
        for (let i = 0; i < iterations; i++) {
            const iterationStart = Date.now();
            // Create test task
            const task = new Task_1.Task('benchmark', `Benchmark task ${i}`, { benchmarkIteration: i }, {}, Task_1.TaskPriority.MEDIUM);
            try {
                await agent.assignTask(task);
                // Wait for task completion (simplified)
                await new Promise(resolve => setTimeout(resolve, 50 + Math.random() * 100));
                const iterationEnd = Date.now();
                const latency = iterationEnd - iterationStart;
                latencies.push(latency);
                completionTimes.push(latency);
                completedTasks++;
            }
            catch (error) {
                failedTasks++;
                logger.warn(`Benchmark iteration ${i} failed:`, error);
            }
        }
        const totalDuration = Date.now() - startTime;
        // Calculate metrics
        const averageLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length;
        const throughput = (completedTasks / totalDuration) * 1000; // tasks per second
        const sortedLatencies = [...latencies].sort((a, b) => a - b);
        const percentiles = {
            p50: sortedLatencies[Math.floor(sortedLatencies.length * 0.5)] || 0,
            p95: sortedLatencies[Math.floor(sortedLatencies.length * 0.95)] || 0,
            p99: sortedLatencies[Math.floor(sortedLatencies.length * 0.99)] || 0
        };
        const taskMetrics = {
            totalTasks: iterations,
            completedTasks,
            failedTasks,
            avgCompletionTime: averageLatency,
            minCompletionTime: Math.min(...completionTimes),
            maxCompletionTime: Math.max(...completionTimes)
        };
        // Compare with baseline if provided
        let comparison;
        if (options.baseline) {
            const latencyDiff = averageLatency - options.baseline.averageLatency;
            const throughputDiff = throughput - options.baseline.throughput;
            const improvement = ((options.baseline.averageLatency - averageLatency) / options.baseline.averageLatency) * 100;
            comparison = {
                improvement: parseFloat(improvement.toFixed(2)),
                latencyDiff: parseFloat(latencyDiff.toFixed(2)),
                throughputDiff: parseFloat(throughputDiff.toFixed(2))
            };
        }
        logger.info(`Benchmark complete: ${averageLatency.toFixed(2)}ms avg, ${throughput.toFixed(2)} tasks/sec`);
        return {
            success: true,
            averageLatency: parseFloat(averageLatency.toFixed(2)),
            throughput: parseFloat(throughput.toFixed(2)),
            taskMetrics,
            percentiles,
            comparison
        };
    }
    catch (error) {
        logger.error('Failed to benchmark agent:', error);
        return {
            success: false,
            averageLatency: 0,
            throughput: 0,
            taskMetrics: {
                totalTasks: 0,
                completedTasks: 0,
                failedTasks: 0,
                avgCompletionTime: 0,
                minCompletionTime: 0,
                maxCompletionTime: 0
            },
            percentiles: { p50: 0, p95: 0, p99: 0 },
            error: error instanceof Error ? error.message : String(error)
        };
    }
}
exports.benchmark = benchmark;
//# sourceMappingURL=benchmark.js.map