openlit
Version:
OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects
255 lines • 11.8 kB
JavaScript
;
/**
* Cross-Language Trace Comparison Utilities
*
* This module provides utilities to compare traces generated by Python and TypeScript
* OpenLIT SDKs to ensure consistency across implementations.
* Uses SemanticConvention keys so comparisons stay in sync with the SDK.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.normalizeTypeScriptSpan = normalizeTypeScriptSpan;
exports.compareTraces = compareTraces;
exports.extractKeyMetrics = extractKeyMetrics;
exports.compareMetrics = compareMetrics;
exports.createTraceValidator = createTraceValidator;
const api_1 = require("@opentelemetry/api");
const semantic_convention_1 = __importDefault(require("../../semantic-convention"));
/**
* Normalize a TypeScript span to a comparable format
*/
function normalizeTypeScriptSpan(span) {
const spanData = span;
return {
spanName: spanData.name || '',
spanKind: spanData.kind?.toString() || 'INTERNAL',
attributes: normalizeAttributes(spanData.attributes || {}),
events: (spanData.events || []).map((event) => ({
name: event.name || '',
attributes: normalizeAttributes(event.attributes || {}),
})),
status: {
code: spanData.status?.code === api_1.SpanStatusCode.OK ? 'OK' :
spanData.status?.code === api_1.SpanStatusCode.ERROR ? 'ERROR' : 'UNSET',
message: spanData.status?.message,
},
duration: spanData.duration?.[0] ? spanData.duration[0] * 1e9 + spanData.duration[1] : undefined,
};
}
/**
* Normalize attributes for comparison (handles array/object differences)
*/
function normalizeAttributes(attrs) {
const normalized = {};
for (const [key, value] of Object.entries(attrs)) {
// Normalize arrays - sort for comparison
if (Array.isArray(value)) {
normalized[key] = [...value].sort();
}
// Normalize numbers (handle float precision)
else if (typeof value === 'number') {
normalized[key] = Math.round(value * 1000) / 1000; // Round to 3 decimal places
}
// Normalize strings (trim whitespace)
else if (typeof value === 'string') {
normalized[key] = value.trim();
}
// Keep other types as-is
else {
normalized[key] = value;
}
}
return normalized;
}
/**
* Compare two normalized traces
*/
function compareTraces(pythonTrace, typescriptTrace) {
const differences = [];
// Compare span names
if (pythonTrace.spanName !== typescriptTrace.spanName) {
differences.push(`Span name mismatch: Python="${pythonTrace.spanName}", TypeScript="${typescriptTrace.spanName}"`);
}
// Compare span kind
if (pythonTrace.spanKind !== typescriptTrace.spanKind) {
differences.push(`Span kind mismatch: Python="${pythonTrace.spanKind}", TypeScript="${typescriptTrace.spanKind}"`);
}
// Compare status
if (pythonTrace.status.code !== typescriptTrace.status.code) {
differences.push(`Status code mismatch: Python="${pythonTrace.status.code}", TypeScript="${typescriptTrace.status.code}"`);
}
// Compare attributes
const attrDiff = compareAttributes(pythonTrace.attributes, typescriptTrace.attributes);
differences.push(...attrDiff);
// Compare events (optional - events may differ slightly)
const eventDiff = compareEvents(pythonTrace.events, typescriptTrace.events);
if (eventDiff.length > 0) {
differences.push(`Event differences: ${eventDiff.join(', ')}`);
}
return {
match: differences.length === 0,
differences,
pythonTrace,
typescriptTrace,
};
}
/**
* Compare attributes between two traces
*/
function compareAttributes(pythonAttrs, typescriptAttrs) {
const differences = [];
const allKeys = new Set([...Object.keys(pythonAttrs), ...Object.keys(typescriptAttrs)]);
// Critical attributes that must match (use SemanticConvention so keys stay in sync)
const criticalAttributes = [
semantic_convention_1.default.GEN_AI_PROVIDER_NAME,
semantic_convention_1.default.GEN_AI_PROVIDER_NAME_OTEL,
semantic_convention_1.default.GEN_AI_OPERATION,
semantic_convention_1.default.GEN_AI_REQUEST_MODEL,
semantic_convention_1.default.GEN_AI_RESPONSE_MODEL,
semantic_convention_1.default.GEN_AI_USAGE_INPUT_TOKENS,
semantic_convention_1.default.GEN_AI_USAGE_OUTPUT_TOKENS,
semantic_convention_1.default.GEN_AI_USAGE_TOTAL_TOKENS,
semantic_convention_1.default.GEN_AI_USAGE_COST,
semantic_convention_1.default.GEN_AI_ENDPOINT,
semantic_convention_1.default.GEN_AI_ENVIRONMENT,
semantic_convention_1.default.GEN_AI_APPLICATION_NAME,
semantic_convention_1.default.SERVER_ADDRESS,
semantic_convention_1.default.SERVER_PORT,
semantic_convention_1.default.GEN_AI_SDK_VERSION,
];
for (const key of allKeys) {
const pythonValue = pythonAttrs[key];
const typescriptValue = typescriptAttrs[key];
const isCritical = criticalAttributes.includes(key);
if (pythonValue === undefined && typescriptValue !== undefined) {
if (isCritical) {
differences.push(`Missing in Python: ${key} (TypeScript has: ${typescriptValue})`);
}
}
else if (typescriptValue === undefined && pythonValue !== undefined) {
if (isCritical) {
differences.push(`Missing in TypeScript: ${key} (Python has: ${pythonValue})`);
}
}
else if (pythonValue !== typescriptValue) {
// For critical attributes, always report differences
if (isCritical) {
differences.push(`Attribute "${key}" mismatch: Python="${pythonValue}", TypeScript="${typescriptValue}"`);
}
// For non-critical, only report if significantly different
else if (typeof pythonValue === 'number' && typeof typescriptValue === 'number') {
const diff = Math.abs(pythonValue - typescriptValue);
if (diff > 0.01) { // Allow small floating point differences
differences.push(`Attribute "${key}" significant difference: Python=${pythonValue}, TypeScript=${typescriptValue}`);
}
}
}
}
return differences;
}
/**
* Compare events between two traces
*/
function compareEvents(pythonEvents, typescriptEvents) {
const differences = [];
if (pythonEvents.length !== typescriptEvents.length) {
differences.push(`Event count mismatch: Python=${pythonEvents.length}, TypeScript=${typescriptEvents.length}`);
}
// Compare event names (order may differ)
const pythonEventNames = pythonEvents.map(e => e.name).sort();
const typescriptEventNames = typescriptEvents.map(e => e.name).sort();
if (JSON.stringify(pythonEventNames) !== JSON.stringify(typescriptEventNames)) {
differences.push(`Event names differ: Python=[${pythonEventNames.join(', ')}], TypeScript=[${typescriptEventNames.join(', ')}]`);
}
return differences;
}
/**
* Extract key metrics from a trace for comparison
*/
function extractKeyMetrics(trace) {
const attrs = trace.attributes;
return {
tokens: {
input: attrs[semantic_convention_1.default.GEN_AI_USAGE_INPUT_TOKENS] || 0,
output: attrs[semantic_convention_1.default.GEN_AI_USAGE_OUTPUT_TOKENS] || 0,
total: attrs[semantic_convention_1.default.GEN_AI_USAGE_TOTAL_TOKENS] || 0,
},
cost: attrs[semantic_convention_1.default.GEN_AI_USAGE_COST] || 0,
model: attrs[semantic_convention_1.default.GEN_AI_REQUEST_MODEL] || attrs[semantic_convention_1.default.GEN_AI_RESPONSE_MODEL] || '',
operation: attrs[semantic_convention_1.default.GEN_AI_OPERATION] || '',
system: attrs[semantic_convention_1.default.GEN_AI_PROVIDER_NAME] || attrs[semantic_convention_1.default.GEN_AI_PROVIDER_NAME_OTEL] || '',
};
}
/**
* Compare key metrics between Python and TypeScript traces
*/
function compareMetrics(pythonTrace, typescriptTrace) {
const pythonMetrics = extractKeyMetrics(pythonTrace);
const typescriptMetrics = extractKeyMetrics(typescriptTrace);
const differences = [];
// Compare tokens (allow small differences due to estimation)
const tokenTolerance = 5; // Allow 5 token difference
if (Math.abs(pythonMetrics.tokens.input - typescriptMetrics.tokens.input) > tokenTolerance) {
differences.push(`Input tokens differ significantly: Python=${pythonMetrics.tokens.input}, TypeScript=${typescriptMetrics.tokens.input}`);
}
if (Math.abs(pythonMetrics.tokens.output - typescriptMetrics.tokens.output) > tokenTolerance) {
differences.push(`Output tokens differ significantly: Python=${pythonMetrics.tokens.output}, TypeScript=${typescriptMetrics.tokens.output}`);
}
// Compare cost (allow small differences due to rounding)
const costTolerance = 0.0001;
if (Math.abs(pythonMetrics.cost - typescriptMetrics.cost) > costTolerance) {
differences.push(`Cost differs: Python=${pythonMetrics.cost}, TypeScript=${typescriptMetrics.cost}`);
}
// Compare model (must match exactly)
if (pythonMetrics.model !== typescriptMetrics.model) {
differences.push(`Model mismatch: Python="${pythonMetrics.model}", TypeScript="${typescriptMetrics.model}"`);
}
// Compare operation (must match exactly)
if (pythonMetrics.operation !== typescriptMetrics.operation) {
differences.push(`Operation mismatch: Python="${pythonMetrics.operation}", TypeScript="${typescriptMetrics.operation}"`);
}
// Compare system (must match exactly)
if (pythonMetrics.system !== typescriptMetrics.system) {
differences.push(`System mismatch: Python="${pythonMetrics.system}", TypeScript="${typescriptMetrics.system}"`);
}
return {
match: differences.length === 0,
differences,
};
}
/**
* Create a test helper that validates trace consistency
*/
function createTraceValidator(providerName, expectedAttributes = []) {
return {
validateTrace: (trace) => {
const errors = [];
// Check required attributes
for (const attr of expectedAttributes) {
if (!(attr in trace.attributes)) {
errors.push(`Missing required attribute: ${attr}`);
}
}
// Check system matches provider (legacy + OTel keys from SemanticConvention)
const providerAttr = trace.attributes[semantic_convention_1.default.GEN_AI_PROVIDER_NAME] ?? trace.attributes[semantic_convention_1.default.GEN_AI_PROVIDER_NAME_OTEL];
if (providerAttr !== providerName) {
errors.push(`System mismatch: expected "${providerName}", got "${providerAttr}"`);
}
// Check operation is set
if (!trace.attributes[semantic_convention_1.default.GEN_AI_OPERATION]) {
errors.push('Missing operation name');
}
// Check tokens are present
if (!trace.attributes[semantic_convention_1.default.GEN_AI_USAGE_INPUT_TOKENS] && !trace.attributes[semantic_convention_1.default.GEN_AI_USAGE_OUTPUT_TOKENS]) {
errors.push('Missing token usage information');
}
return {
valid: errors.length === 0,
errors,
};
},
};
}
//# sourceMappingURL=trace-comparison-utils.js.map