@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
605 lines • 22 kB
JavaScript
/**
* Pipeline Tracer - Expert-level pipeline tracing and profiling
*
* Provides comprehensive request tracing through all 4 consensus stages,
* performance bottleneck identification, and detailed profiling capabilities.
*/
import { v4 as uuidv4 } from 'uuid';
import { structuredLogger } from './structured-logger.js';
export class PipelineTracer {
traces = new Map();
activeSpans = new Map();
config;
cleanupTimer;
constructor(config) {
this.config = {
enabled: true,
samplingRate: 1.0, // Trace everything by default
maxTraces: 1000,
retentionDays: 7,
detailLevel: 'standard',
components: {
consensus: true,
openrouter: true,
database: true,
validator: true,
system: true
},
profiling: {
enabled: true,
memoryTracking: true,
cpuProfiling: false, // CPU profiling can be expensive
stackTraces: false
},
...config
};
this.startCleanupTimer();
}
/**
* Start a new trace
*/
startTrace(operationType, operationName, options) {
if (!this.config.enabled || !this.shouldSample()) {
return null;
}
const traceId = uuidv4();
const now = performance.now();
// Create root span
const rootSpan = {
id: uuidv4(),
traceId,
operationName,
component: 'consensus',
startTime: now,
tags: options?.tags || {},
logs: [],
status: 'active',
metrics: {}
};
// Create trace
const trace = {
id: traceId,
conversationId: options?.conversationId,
operationType,
startTime: now,
spans: [rootSpan],
rootSpan,
user: options?.user,
environment: process.env.NODE_ENV || 'development',
tags: options?.tags || {},
performance: {
criticalPath: [],
bottlenecks: [],
efficiency: 0,
recommendations: []
}
};
this.traces.set(traceId, trace);
this.activeSpans.set(rootSpan.id, rootSpan);
structuredLogger.debug('Trace started', {
traceId,
operationType,
operationName,
conversationId: options?.conversationId
});
return traceId;
}
/**
* Start a new span within a trace
*/
startSpan(traceId, operationName, component, options) {
const trace = this.traces.get(traceId);
if (!trace || !this.isComponentEnabled(component)) {
return null;
}
const spanId = uuidv4();
const now = performance.now();
const span = {
id: spanId,
traceId,
parentId: options?.parentId,
operationName,
component,
startTime: now,
tags: options?.tags || {},
logs: [],
status: 'active',
metrics: this.config.profiling.enabled ? this.captureInitialMetrics() : {}
};
trace.spans.push(span);
this.activeSpans.set(spanId, span);
structuredLogger.debug('Span started', {
traceId,
spanId,
operationName,
component,
parentId: options?.parentId
});
return spanId;
}
/**
* Finish a span
*/
finishSpan(spanId, status = 'success', error) {
const span = this.activeSpans.get(spanId);
if (!span)
return;
const now = performance.now();
span.endTime = now;
span.duration = now - span.startTime;
span.status = status;
span.error = error;
// Capture final metrics if profiling enabled
if (this.config.profiling.enabled) {
span.metrics = { ...span.metrics, ...this.captureFinalMetrics(span.metrics) };
}
this.activeSpans.delete(spanId);
structuredLogger.debug('Span finished', {
traceId: span.traceId,
spanId,
operationName: span.operationName,
duration: span.duration,
status
});
// If this is the root span, finish the trace
const trace = this.traces.get(span.traceId);
if (trace && span.id === trace.rootSpan.id) {
this.finishTrace(span.traceId);
}
}
/**
* Add a log entry to a span
*/
addSpanLog(spanId, level, message, fields) {
const span = this.activeSpans.get(spanId);
if (!span)
return;
span.logs.push({
timestamp: performance.now(),
level,
message,
fields
});
if (this.config.detailLevel === 'verbose') {
structuredLogger.debug('Span log added', {
traceId: span.traceId,
spanId,
level,
message
});
}
}
/**
* Set span tags
*/
setSpanTags(spanId, tags) {
const span = this.activeSpans.get(spanId);
if (!span)
return;
span.tags = { ...span.tags, ...tags };
}
/**
* Finish a trace and perform analysis
*/
finishTrace(traceId) {
const trace = this.traces.get(traceId);
if (!trace)
return;
const now = performance.now();
trace.endTime = now;
trace.totalDuration = now - trace.startTime;
// Perform trace analysis
this.analyzeTrace(trace);
structuredLogger.info('Trace completed', {
traceId,
operationType: trace.operationType,
totalDuration: trace.totalDuration,
spanCount: trace.spans.length,
efficiency: trace.performance.efficiency,
bottleneckCount: trace.performance.bottlenecks.length
});
// Clean up if we exceed max traces
this.enforceTraceLimit();
}
/**
* Analyze trace performance and identify bottlenecks
*/
analyzeTrace(trace) {
// Calculate critical path
trace.performance.criticalPath = this.calculateCriticalPath(trace);
// Identify bottlenecks
trace.performance.bottlenecks = this.identifyBottlenecks(trace);
// Calculate efficiency
trace.performance.efficiency = this.calculateEfficiency(trace);
// Generate recommendations
trace.performance.recommendations = this.generateRecommendations(trace);
}
/**
* Calculate the critical path through the trace
*/
calculateCriticalPath(trace) {
const spanMap = new Map(trace.spans.map(span => [span.id, span]));
const criticalPath = [];
// Find spans that are on the critical path (longest execution path)
const processSpan = (spanId, visited) => {
if (visited.has(spanId))
return 0;
visited.add(spanId);
const span = spanMap.get(spanId);
if (!span || !span.duration)
return 0;
const children = trace.spans.filter(s => s.parentId === spanId);
let maxChildDuration = 0;
for (const child of children) {
const childDuration = processSpan(child.id, visited);
if (childDuration > maxChildDuration) {
maxChildDuration = childDuration;
}
}
const totalDuration = span.duration + maxChildDuration;
criticalPath.push(span.operationName);
return totalDuration;
};
processSpan(trace.rootSpan.id, new Set());
return criticalPath;
}
/**
* Identify performance bottlenecks
*/
identifyBottlenecks(trace) {
const bottlenecks = [];
const totalDuration = trace.totalDuration || 1;
for (const span of trace.spans) {
if (!span.duration)
continue;
const percentageOfTotal = (span.duration / totalDuration) * 100;
// Consider spans that take more than 20% of total time as potential bottlenecks
if (percentageOfTotal > 20) {
const impact = this.categorizeImpact(percentageOfTotal);
const analysis = {
spanId: span.id,
operationName: span.operationName,
component: span.component,
impact,
duration: span.duration,
percentageOfTotal,
cause: this.identifyBottleneckCause(span),
recommendation: this.generateBottleneckRecommendation(span)
};
bottlenecks.push(analysis);
}
}
return bottlenecks.sort((a, b) => b.percentageOfTotal - a.percentageOfTotal);
}
/**
* Calculate trace efficiency
*/
calculateEfficiency(trace) {
if (!trace.totalDuration || trace.spans.length === 0)
return 0;
// Calculate efficiency based on parallelization and wait times
const activeTime = trace.spans.reduce((sum, span) => sum + (span.duration || 0), 0);
const totalTime = trace.totalDuration * trace.spans.length;
return totalTime > 0 ? (activeTime / totalTime) * 100 : 0;
}
/**
* Generate performance recommendations
*/
generateRecommendations(trace) {
const recommendations = [];
// Check for sequential operations that could be parallelized
const sequentialSpans = this.findSequentialSpans(trace);
if (sequentialSpans.length > 2) {
recommendations.push('Consider parallelizing sequential operations to reduce total execution time');
}
// Check for high-latency operations
const slowSpans = trace.spans.filter(span => (span.duration || 0) > 5000);
if (slowSpans.length > 0) {
recommendations.push('Optimize slow operations or implement caching for frequently accessed data');
}
// Check for retry patterns
const retryableSpans = trace.spans.filter(span => span.operationName.includes('retry') || span.logs.some(log => log.message.includes('retry')));
if (retryableSpans.length > 0) {
recommendations.push('Review retry policies and implement circuit breakers for failing services');
}
// Check for memory usage
const highMemorySpans = trace.spans.filter(span => span.metrics.memoryUsage && span.metrics.memoryUsage > 100 * 1024 * 1024 // 100MB
);
if (highMemorySpans.length > 0) {
recommendations.push('Optimize memory usage in high-consumption operations');
}
return recommendations;
}
/**
* Get trace by ID
*/
getTrace(traceId) {
return this.traces.get(traceId);
}
/**
* Get all traces with optional filtering
*/
getTraces(filter) {
let traces = Array.from(this.traces.values());
if (filter) {
if (filter.operationType) {
traces = traces.filter(t => t.operationType === filter.operationType);
}
if (filter.conversationId) {
traces = traces.filter(t => t.conversationId === filter.conversationId);
}
if (filter.user) {
traces = traces.filter(t => t.user === filter.user);
}
if (filter.since) {
const sinceMs = filter.since.getTime();
traces = traces.filter(t => t.startTime >= sinceMs);
}
if (filter.status) {
traces = traces.filter(t => {
const isCompleted = t.endTime !== undefined;
const hasErrors = t.spans.some(s => s.status === 'error');
switch (filter.status) {
case 'active':
return !isCompleted;
case 'completed':
return isCompleted && !hasErrors;
case 'error':
return hasErrors;
default:
return true;
}
});
}
}
return traces.sort((a, b) => b.startTime - a.startTime);
}
/**
* Get trace statistics
*/
getTracingStats() {
const traces = Array.from(this.traces.values());
const completedTraces = traces.filter(t => t.endTime !== undefined);
const totalTraces = traces.length;
const activeTraces = traces.length - completedTraces.length;
const averageDuration = completedTraces.length > 0
? completedTraces.reduce((sum, t) => sum + (t.totalDuration || 0), 0) / completedTraces.length
: 0;
const averageEfficiency = completedTraces.length > 0
? completedTraces.reduce((sum, t) => sum + t.performance.efficiency, 0) / completedTraces.length
: 0;
// Component breakdown
const componentBreakdown = {};
traces.forEach(trace => {
trace.spans.forEach(span => {
componentBreakdown[span.component] = (componentBreakdown[span.component] || 0) + 1;
});
});
// Bottleneck frequency
const bottleneckFrequency = {};
traces.forEach(trace => {
trace.performance.bottlenecks.forEach(bottleneck => {
const key = `${bottleneck.component}:${bottleneck.operationName}`;
bottleneckFrequency[key] = (bottleneckFrequency[key] || 0) + 1;
});
});
return {
totalTraces,
activeTraces,
averageDuration,
averageEfficiency,
componentBreakdown,
bottleneckFrequency
};
}
/**
* Export trace data
*/
exportTraces(format = 'json') {
const traces = Array.from(this.traces.values());
switch (format) {
case 'json':
return JSON.stringify(traces, null, 2);
case 'jaeger':
return this.formatAsJaeger(traces);
case 'zipkin':
return this.formatAsZipkin(traces);
default:
throw new Error(`Unsupported export format: ${format}`);
}
}
/**
* Helper methods
*/
shouldSample() {
return Math.random() < this.config.samplingRate;
}
isComponentEnabled(component) {
return this.config.components[component] || false;
}
captureInitialMetrics() {
const metrics = {};
if (this.config.profiling.memoryTracking) {
const memUsage = process.memoryUsage();
metrics.memoryUsage = memUsage.heapUsed;
}
return metrics;
}
captureFinalMetrics(initialMetrics) {
const metrics = {};
if (this.config.profiling.memoryTracking && initialMetrics.memoryUsage) {
const currentMemUsage = process.memoryUsage().heapUsed;
metrics.memoryUsage = currentMemUsage - initialMetrics.memoryUsage;
}
return metrics;
}
categorizeImpact(percentage) {
if (percentage > 50)
return 'critical';
if (percentage > 35)
return 'high';
if (percentage > 20)
return 'medium';
return 'low';
}
identifyBottleneckCause(span) {
// Analyze span characteristics to identify likely cause
if (span.component === 'openrouter') {
return 'High API latency or rate limiting';
}
if (span.component === 'database') {
return 'Slow database queries or connection issues';
}
if (span.operationName.includes('validation')) {
return 'Complex validation logic or large data processing';
}
if (span.metrics.memoryUsage && span.metrics.memoryUsage > 50 * 1024 * 1024) {
return 'High memory usage indicating data processing bottleneck';
}
return 'General processing delay';
}
generateBottleneckRecommendation(span) {
if (span.component === 'openrouter') {
return 'Consider using faster models, implementing caching, or parallel requests';
}
if (span.component === 'database') {
return 'Optimize queries, add indexes, or implement connection pooling';
}
if (span.operationName.includes('validation')) {
return 'Optimize validation logic or implement early validation exit';
}
return 'Profile the operation to identify specific performance issues';
}
findSequentialSpans(trace) {
// Find spans that are executed sequentially rather than in parallel
const sequentialSpans = [];
const parentMap = new Map();
// Group spans by parent
trace.spans.forEach(span => {
const parentId = span.parentId || 'root';
if (!parentMap.has(parentId)) {
parentMap.set(parentId, []);
}
parentMap.get(parentId).push(span);
});
// Find groups with sequential execution
parentMap.forEach(siblings => {
if (siblings.length > 1) {
const sorted = siblings.sort((a, b) => a.startTime - b.startTime);
sequentialSpans.push(...sorted);
}
});
return sequentialSpans;
}
formatAsJaeger(traces) {
// Format traces for Jaeger (simplified version)
const jaegerTraces = traces.map(trace => ({
traceID: trace.id,
spans: trace.spans.map(span => ({
traceID: trace.id,
spanID: span.id,
parentSpanID: span.parentId,
operationName: span.operationName,
startTime: Math.floor(span.startTime * 1000), // Convert to microseconds
duration: Math.floor((span.duration || 0) * 1000),
tags: Object.entries(span.tags).map(([key, value]) => ({
key,
type: typeof value === 'string' ? 'string' : 'number',
value: value.toString()
})),
logs: span.logs.map(log => ({
timestamp: Math.floor(log.timestamp * 1000),
fields: [
{ key: 'level', value: log.level },
{ key: 'message', value: log.message },
...(log.fields ? Object.entries(log.fields).map(([k, v]) => ({ key: k, value: String(v) })) : [])
]
}))
}))
}));
return JSON.stringify({ data: jaegerTraces }, null, 2);
}
formatAsZipkin(traces) {
// Format traces for Zipkin (simplified version)
const zipkinSpans = traces.flatMap(trace => trace.spans.map(span => ({
traceId: trace.id.replace(/-/g, ''),
id: span.id.replace(/-/g, ''),
parentId: span.parentId?.replace(/-/g, ''),
name: span.operationName,
timestamp: Math.floor(span.startTime * 1000),
duration: Math.floor((span.duration || 0) * 1000),
localEndpoint: {
serviceName: 'hive-ai',
ipv4: '127.0.0.1'
},
tags: Object.fromEntries(Object.entries(span.tags).map(([key, value]) => [key, String(value)])),
annotations: span.logs.map(log => ({
timestamp: Math.floor(log.timestamp * 1000),
value: log.message
}))
})));
return JSON.stringify(zipkinSpans, null, 2);
}
startCleanupTimer() {
// Clean up old traces every hour
this.cleanupTimer = setInterval(() => {
this.cleanupOldTraces();
}, 60 * 60 * 1000);
}
cleanupOldTraces() {
const cutoffTime = Date.now() - (this.config.retentionDays * 24 * 60 * 60 * 1000);
const toDelete = [];
this.traces.forEach((trace, traceId) => {
if (trace.startTime < cutoffTime) {
toDelete.push(traceId);
}
});
toDelete.forEach(traceId => {
this.traces.delete(traceId);
});
if (toDelete.length > 0) {
structuredLogger.info('Cleaned up old traces', { deletedCount: toDelete.length });
}
}
enforceTraceLimit() {
if (this.traces.size <= this.config.maxTraces)
return;
// Remove oldest traces
const traces = Array.from(this.traces.entries())
.sort((a, b) => a[1].startTime - b[1].startTime);
const toDelete = traces.slice(0, traces.length - this.config.maxTraces);
toDelete.forEach(([traceId]) => {
this.traces.delete(traceId);
});
structuredLogger.info('Enforced trace limit', { deletedCount: toDelete.length });
}
/**
* Update tracing configuration
*/
updateConfig(newConfig) {
this.config = { ...this.config, ...newConfig };
structuredLogger.info('Tracing configuration updated');
}
/**
* Get current configuration
*/
getConfig() {
return { ...this.config };
}
/**
* Cleanup and stop tracing
*/
shutdown() {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
}
this.traces.clear();
this.activeSpans.clear();
structuredLogger.info('Pipeline tracer shutdown completed');
}
}
/**
* Global pipeline tracer instance
*/
export const globalPipelineTracer = new PipelineTracer();
//# sourceMappingURL=pipeline-tracer.js.map