@erickluis00/otelviewer
Version:
Shared OpenTelemetry tracing utilities, types, and batch processor for Realtime OpenTelemetry Viewer [WIP]
389 lines • 17 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RealtimeSpanProcessor = void 0;
const api_1 = require("@opentelemetry/api");
const config_1 = require("./config");
// Converts OpenTelemetry HRTime [seconds, nanoseconds] to milliseconds
function hrTimeToMilliseconds(hrTime) {
// Ensure we return an integer timestamp to avoid PostgreSQL BIGINT issues
return Math.round(hrTime[0] * 1000 + hrTime[1] / 1e6);
}
/**
* Realtime span processor that collects spans and sends them to an ingest endpoint
* Supports real-time span sending and batched snapshots with exponential backoff retry
*/
class RealtimeSpanProcessor {
constructor(config = {}) {
this.inMemoryStore = new Map(); // traceId -> spans[]
this.changedSpans = new Map(); // traceId -> Set<spanId>
this.rolledBackSpans = []; // Queue of spans to be sent
this.snapshotTimer = null;
// Retry mechanism state
this.retryCount = 0;
this.maxRetries = 5;
this.baseRetryDelay = 1000; // 1 second base delay
this.maxRetryDelay = 30000; // 30 seconds max delay
this.nextRetryTime = 0;
// Recovery mode state (after max retries exhausted)
this.isInRecoveryMode = false;
this.recoveryInterval = 60000; // 1 minute recovery attempts
this.nextRecoveryTime = 0;
this.config = (0, config_1.createConfig)(config);
console.log(`🏷️ OTEL Project ID: ${this.config.projectId}`);
this.startSnapshotTimer();
}
startSnapshotTimer() {
this.snapshotTimer = setInterval(async () => {
await this.sendSnapshot();
}, this.config.snapshotInterval);
// console.log(`Started snapshot processor with ${this.config.snapshotInterval}ms interval`);
}
/**
* Calculate exponential backoff delay with jitter
*/
calculateRetryDelay() {
const exponentialDelay = this.baseRetryDelay * Math.pow(2, this.retryCount);
const jitter = Math.random() * 0.1 * exponentialDelay; // Add 10% jitter
const delay = Math.min(exponentialDelay + jitter, this.maxRetryDelay);
return Math.round(delay);
}
/**
* Reset retry state after successful send
*/
resetRetryState() {
this.retryCount = 0;
this.nextRetryTime = 0;
this.isInRecoveryMode = false;
this.nextRecoveryTime = 0;
}
/**
* Enter recovery mode after max retries exhausted
*/
enterRecoveryMode() {
this.isInRecoveryMode = true;
this.retryCount = 0;
this.nextRetryTime = 0;
this.nextRecoveryTime = Date.now() + this.recoveryInterval;
// Clear memory store to prevent memory leak
const tracesCleared = this.inMemoryStore.size;
this.inMemoryStore.clear();
this.changedSpans.clear();
console.log(`🚑 [OTEL Recovery] Entering recovery mode. Cleared ${tracesCleared} traces from memory. Will try every ${Math.round(this.recoveryInterval / 1000)}s indefinitely.`);
}
/**
* Log detailed error information for debugging
*/
logDetailedError(error, context) {
console.error(`❌ [OTEL Error] ${context}:`, {
message: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined,
timestamp: new Date().toISOString(),
retryCount: this.retryCount,
maxRetries: this.maxRetries,
isInRecoveryMode: this.isInRecoveryMode,
pendingSpans: this.rolledBackSpans.length
});
}
onStart(span, parentContext) {
const sdkSpan = span;
// console.log(`InMemoryBatchProcessor.onStart: ${sdkSpan.name} (spanId=${sdkSpan.spanContext().spanId}, traceId=${sdkSpan.spanContext().traceId})`);
// Merge resource attributes with span attributes
const mergedAttributes = {
...sdkSpan.resource.attributes,
...sdkSpan.attributes,
};
// Ensure deployment.environment.name is always set
if (!mergedAttributes['deployment.environment.name']) {
mergedAttributes['deployment.environment.name'] = typeof process !== 'undefined' && process.env?.NODE_ENV
? process.env.NODE_ENV
: 'INVALID NODE_ENV';
}
const partialSpan = this.convertPartialSpan(sdkSpan.spanContext(), sdkSpan.name, sdkSpan.startTime, mergedAttributes, sdkSpan.parentSpanContext?.spanId);
this.addSpanToMemoryStore(partialSpan);
}
onEnd(span) {
const completeSpans = this.convertCompleteSpan(span);
const traceId = span.spanContext().traceId;
// Add all complete spans (span + events as logs) to memory store
completeSpans.forEach(spanOrLog => {
this.addSpanToMemoryStore(spanOrLog);
});
}
addSpanToMemoryStore(span) {
const traceId = span.traceId;
// In recovery mode, do not add spans to memory store
if (this.isInRecoveryMode) {
return;
}
if (!this.inMemoryStore.has(traceId)) {
this.inMemoryStore.set(traceId, []);
}
const spans = this.inMemoryStore.get(traceId);
// Check if span already exists (update scenario - partial to complete)
const existingIndex = spans.findIndex(s => s.id === span.id);
if (existingIndex !== -1) {
spans[existingIndex] = span;
}
else {
spans.push(span);
}
// add to changed spans
if (!this.changedSpans.has(traceId)) {
this.changedSpans.set(traceId, new Set());
}
this.changedSpans.get(traceId).add(span.id);
}
async sendSnapshot() {
const now = Date.now();
// Check if we should wait for retry delay (normal retry mode)
if (!this.isInRecoveryMode && this.nextRetryTime > 0 && now < this.nextRetryTime) {
return;
}
// Check if we're in recovery mode and should wait for recovery interval
if (this.isInRecoveryMode && now < this.nextRecoveryTime) {
return;
}
// FIRST: Clone changedSpans to avoid racing and outdated data
const clonedChangedSpans = new Map();
for (const [traceId, spanIds] of this.changedSpans.entries()) {
clonedChangedSpans.set(traceId, new Set(spanIds));
}
// SECOND: Clean the original changed spans
this.changedSpans.clear();
// THIRD: Collect all pending spans from the queue
const spansToSend = [...this.rolledBackSpans];
this.rolledBackSpans = []; // Clear the queue
// FOURTH: Based on cloned instance, add changed spans to the spans array
for (const [traceId, spanIds] of clonedChangedSpans.entries()) {
const traceSpans = this.inMemoryStore.get(traceId);
if (traceSpans) {
for (const spanId of spanIds) {
const span = traceSpans.find(s => s.id === spanId);
if (span) {
spansToSend.push(span);
}
}
}
}
// Clean up completed traces from memory store
this.cleanupCompletedTraces();
if (spansToSend.length === 0) {
return;
}
let attemptType;
let attemptIcon;
let attemptInfo;
if (this.isInRecoveryMode) {
attemptType = 'Recovery';
attemptIcon = '🚑';
attemptInfo = `recovery attempt`;
}
else if (this.retryCount > 0) {
attemptType = 'Retry';
attemptIcon = '🔄';
attemptInfo = `attempt ${this.retryCount + 1}/${this.maxRetries + 1}`;
}
else {
attemptType = 'Send';
attemptIcon = '📤';
attemptInfo = `initial attempt`;
}
if (process.env.NODE_ENV !== 'production' && (this.isInRecoveryMode || this.retryCount > 0)) {
console.log(`${attemptIcon} [OTEL ${attemptType}] Attempting to send ${spansToSend.length} spans (${attemptInfo})`);
}
try {
const startTime = Date.now();
const res = await fetch(this.config.ingestUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
...(this.config.ingestSecret ? { 'X-Ingest-Secret': this.config.ingestSecret } : {}),
},
body: JSON.stringify({ spans: spansToSend }),
});
if (!res.ok) {
const text = await res.text().catch(() => 'Failed to read response body');
const errorDetails = {
status: res.status,
statusText: res.statusText,
responseBody: text,
url: this.config.ingestUrl,
headers: Object.fromEntries(res.headers.entries())
};
throw new Error(`HTTP ${res.status} ${res.statusText}: ${text}`);
}
const duration = Date.now() - startTime;
if (this.isInRecoveryMode) {
console.log(`🎉 [OTEL Recovered] Service is back online! Sent ${spansToSend.length} spans in ${duration}ms`);
}
else {
// if (process.env.NODE_ENV !== 'production') {
// console.log(`✅ [OTEL Success] Sent ${spansToSend.length} spans in ${duration}ms (${this.config.ingestUrl})`);
// }
// console.log(`✅ [OTEL Success] Sent ${spansToSend.length} spans in ${duration}ms (${this.config.ingestUrl})`);
}
// Reset retry state on successful send
this.resetRetryState();
}
catch (error) {
this.logDetailedError(error, 'Failed to send spans to ingest endpoint: ' + this.config.ingestUrl);
if (this.isInRecoveryMode) {
// In recovery mode - explicitly discard spans and schedule next recovery attempt
console.log(`🗑️ [OTEL Recovery] Service still down. Discarding ${spansToSend.length} spans, next recovery attempt in ${Math.round(this.recoveryInterval / 1000)}s`);
// Spans are discarded here (not added back to rolledBackSpans)
this.nextRecoveryTime = Date.now() + this.recoveryInterval;
}
else if (this.retryCount < this.maxRetries) {
// Normal retry mode - rollback spans and exponential backoff
this.rolledBackSpans.unshift(...spansToSend);
this.retryCount++;
const retryDelay = this.calculateRetryDelay();
this.nextRetryTime = Date.now() + retryDelay;
console.log(`⏳ [OTEL Retry] Scheduling retry ${this.retryCount}/${this.maxRetries} in ${Math.round(retryDelay / 1000)}s`);
}
else {
// Max retries exhausted - enter recovery mode
this.rolledBackSpans.unshift(...spansToSend);
console.warn(`⚠️ [OTEL] Max retries exhausted. Entering recovery mode - spans will be discarded until service recovers.`);
this.enterRecoveryMode();
}
}
}
/**
* Checks all traces in Memory Store and removes completed traces.
* A trace is considered completed when its root span (parentId is null) is completed (not partial).
*/
cleanupCompletedTraces() {
for (const [traceId, spans] of this.inMemoryStore.entries()) {
// Find the root span (parentId is null)
const rootSpan = spans.find(span => span.parentId === null && span.kind === 'SPAN');
if (rootSpan && !rootSpan.isPartial) {
// Root span is completed, remove the entire trace from memory store
this.inMemoryStore.delete(traceId);
// Also clean up any remaining entries in changedSpans for this trace
this.changedSpans.delete(traceId);
// if (process.env.NODE_ENV !== 'production') {
// console.log(`🧹 Cleaned up completed trace ${traceId} with ${spans.length} spans..`);
// }
}
}
}
async forceFlush() {
// Reset retry state to force immediate send
this.resetRetryState();
await this.sendSnapshot();
}
/**
* Get current processor status for debugging
*/
getStatus() {
return {
isInRecoveryMode: this.isInRecoveryMode,
retryCount: this.retryCount,
pendingSpans: this.rolledBackSpans.length,
nextRetryTime: this.nextRetryTime,
nextRecoveryTime: this.nextRecoveryTime
};
}
async shutdown() {
// Stop the snapshot timer
if (this.snapshotTimer) {
clearInterval(this.snapshotTimer);
this.snapshotTimer = null;
}
// Reset retry state to allow final send attempt
this.resetRetryState();
// Final snapshot send
await this.sendSnapshot();
// Log final status
const status = this.getStatus();
if (status.pendingSpans > 0) {
console.warn(`⚠️ [OTEL Shutdown] ${status.pendingSpans} spans remaining in queue`);
}
console.log('InMemoryBatchProcessor shutdown complete');
}
convertPartialSpan(spanContext, name, startTime, attributes, parentSpanId) {
const startTimeMs = hrTimeToMilliseconds(startTime);
return {
id: spanContext.spanId,
kind: 'SPAN',
traceId: spanContext.traceId,
parentId: parentSpanId ?? null,
name: name,
startTime: startTimeMs,
endTime: startTimeMs, // Set to start time initially
status: { code: 'UNSET' },
attributes: attributes,
projectId: this.config.projectId,
isPartial: true,
isRealTime: false,
};
}
convertCompleteSpan(span) {
const traceId = span.spanContext().traceId;
const spans = [];
// Merge resource attributes with span attributes
const mergedAttributes = {
...span.resource.attributes,
...span.attributes,
};
// Ensure deployment.environment.name is always set
if (!mergedAttributes['deployment.environment.name']) {
mergedAttributes['deployment.environment.name'] = typeof process !== 'undefined' && process.env?.NODE_ENV
? process.env.NODE_ENV
: 'INVALID NODE_ENV';
}
// Main span
const customSpan = {
id: span.spanContext().spanId,
kind: 'SPAN',
traceId: traceId,
parentId: span.parentSpanContext?.spanId ?? null,
name: span.name,
startTime: hrTimeToMilliseconds(span.startTime),
endTime: hrTimeToMilliseconds(span.endTime),
status: {
code: span.status.code === api_1.SpanStatusCode.ERROR ? 'ERROR' : 'OK',
message: span.status.message,
},
attributes: mergedAttributes,
projectId: this.config.projectId,
isPartial: false,
isRealTime: false,
};
// console.log(`[otel] convertCompleteSpan: Main span: ${JSON.stringify(customSpan, null, 2)}`);
spans.push(customSpan);
// Convert events to log entries
span.events.forEach((event, index) => {
const isException = event.name === 'exception' ||
event.attributes?.['exception.type'] ||
event.attributes?.['exception.message'];
const eventTimeMs = hrTimeToMilliseconds(event.time);
const log = {
id: `${span.spanContext().spanId}-event-${index}`,
kind: 'LOG',
traceId: traceId,
parentId: span.spanContext().spanId,
name: event.name,
startTime: eventTimeMs,
endTime: eventTimeMs,
status: {
code: isException ? 'ERROR' : 'OK',
message: isException ? String(event.attributes?.['exception.message'] || 'Exception occurred') : undefined
},
attributes: {
'log.severity': isException ? 'ERROR' : (event.attributes?.['log.severity'] || 'INFO'),
'log.message': event.name,
...event.attributes,
},
projectId: this.config.projectId,
isPartial: false,
isRealTime: false,
};
spans.push(log);
});
return spans;
}
}
exports.RealtimeSpanProcessor = RealtimeSpanProcessor;
//# sourceMappingURL=batch-processor.js.map