@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
229 lines (228 loc) • 9.72 kB
JavaScript
/**
* Telemetry Handler Module
*
* Handles analytics, evaluation, performance metrics, and telemetry configuration.
* Extracted from BaseProvider to follow Single Responsibility Principle.
*
* Responsibilities:
* - Analytics creation and tracking
* - Evaluation generation
* - Performance metrics recording
* - Cost calculation
* - Telemetry configuration
*
* @module core/modules/TelemetryHandler
*/
import { nanoid } from "nanoid";
import { extractTokenUsage } from "../../utils/tokenUtils.js";
import { logger } from "../../utils/logger.js";
import { recordProviderPerformanceFromMetrics } from "../evaluationProviders.js";
import { modelConfig } from "../modelConfiguration.js";
import { TelemetryService } from "../../telemetry/telemetryService.js";
import { calculateCost, hasPricing } from "../../utils/pricing.js";
import { getLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
/**
* TelemetryHandler class - Handles analytics and telemetry for AI providers
*/
export class TelemetryHandler {
providerName;
modelName;
neurolink;
constructor(providerName, modelName, neurolink) {
this.providerName = providerName;
this.modelName = modelName;
this.neurolink = neurolink;
}
/**
* Create analytics for a generation result
*/
async createAnalytics(result, responseTime, context) {
const { createAnalytics } = await import("../analytics.js");
return createAnalytics(this.providerName, this.modelName, result, responseTime, context);
}
/**
* Create evaluation for a generation result
*/
async createEvaluation(result, options) {
const { evaluateResponse } = await import("../evaluation.js");
const context = {
userQuery: options.prompt || options.input?.text || "Generated response",
aiResponse: result.content,
context: options.context,
primaryDomain: options.evaluationDomain,
assistantRole: "AI assistant",
conversationHistory: options.conversationHistory?.map((msg) => ({
role: msg.role,
content: msg.content,
})),
toolUsage: options.toolUsageContext
? [
{
toolName: options.toolUsageContext,
input: {},
output: {},
executionTime: 0,
},
]
: undefined,
expectedOutcome: options.expectedOutcome,
evaluationCriteria: options.evaluationCriteria,
};
const evaluation = await evaluateResponse(context);
return evaluation;
}
/**
* Record performance metrics for a generation
*/
async recordPerformanceMetrics(usage, responseTime) {
try {
// Normalize first: rebases cache-inclusive ai@6 input onto the uncached
// remainder and surfaces cache tiers so the cost is priced per tier
// instead of billing cache reads at the full input rate.
const tokenUsage = extractTokenUsage(usage);
const totalTokens = tokenUsage.total;
const actualCost = await this.calculateActualCost(tokenUsage);
recordProviderPerformanceFromMetrics(this.providerName, {
responseTime,
tokensGenerated: totalTokens,
cost: actualCost,
success: true,
});
// Wire TelemetryService metrics so OTEL counters/histograms are populated
TelemetryService.getInstance().recordAIRequest(this.providerName, this.modelName, totalTokens, responseTime, actualCost > 0 ? actualCost : undefined);
logger.debug(`Performance recorded for ${this.providerName}`, {
responseTime: `${responseTime}ms`,
tokens: totalTokens,
estimatedCost: `$${actualCost.toFixed(6)}`,
});
}
catch (perfError) {
logger.warn("⚠️ Performance recording failed:", perfError);
}
}
/**
* Record a TTS synthesis failure without affecting the generation result.
*/
recordTTSFailure(ttsProvider, error, latency) {
try {
const labels = {
provider: ttsProvider,
error_code: error.code,
...(error.retriable !== undefined
? { retriable: error.retriable.toString() }
: {}),
};
const telemetry = TelemetryService.getInstance();
telemetry.recordCustomMetric("tts_failures", 1, labels);
telemetry.recordCustomHistogram("tts_failure_latency_ms", latency, labels);
}
catch (telemetryError) {
logger.warn("TTS failure telemetry recording failed:", telemetryError);
}
}
/**
* Calculate actual cost based on token usage and provider configuration.
*
* Uses the per-model pricing table first (which has accurate rates for
* specific models like Claude on Vertex AI), then falls back to the
* provider-level default cost from modelConfiguration.
*
* Previously this only used modelConfig.getCostInfo() which returns
* provider-level defaults (e.g. Gemini rates for the "vertex" provider),
* causing a ~1,780x under-estimate when the actual model was Claude Sonnet
* on Vertex AI ($0.000060 vs $0.106895 for the same request).
*/
async calculateActualCost(usage) {
try {
// Try the per-model pricing table first (includes correct rates for
// Claude on Vertex, cache token rates, etc.)
if (hasPricing(this.providerName, this.modelName)) {
return calculateCost(this.providerName, this.modelName, usage);
}
// Fall back to provider-level default cost from configuration system
const costInfo = modelConfig.getCostInfo(this.providerName, this.modelName);
if (!costInfo) {
return 0; // No cost info available
}
// Calculate cost per 1K tokens. costInfo has no cache tiers, so cache
// tokens are billed at the input rate — the same total a provider
// without cache-aware splitting would have reported, never $0.
const inputCost = ((usage.input +
(usage.cacheReadTokens ?? 0) +
(usage.cacheCreationTokens ?? 0)) /
1000) *
costInfo.input;
const outputCost = (usage.output / 1000) * costInfo.output;
return inputCost + outputCost;
}
catch (error) {
logger.debug(`Cost calculation failed for ${this.providerName}:`, error);
return 0; // Fallback to 0 on any error
}
}
/**
* Create telemetry configuration for Vercel AI SDK experimental_telemetry
* This enables automatic OpenTelemetry tracing when telemetry is enabled
*/
getTelemetryConfig(options, operationType = "stream") {
// Check if telemetry is enabled via NeuroLink observability config
if (!this.neurolink?.isTelemetryEnabled()) {
return undefined;
}
const context = options.context;
const langfuseContext = getLangfuseContext();
const traceName = context?.traceName ?? langfuseContext?.traceName;
const userId = context?.userId ?? langfuseContext?.userId;
const functionId = traceName ? traceName : userId ? userId : "guest";
const metadata = {
...(context?.metadata || {}),
provider: this.providerName,
model: this.modelName,
toolsEnabled: !options.disableTools,
neurolink: true,
operationType,
originalProvider: this.providerName,
};
// Add sessionId if available
if ("sessionId" in options && options.sessionId) {
const sessionId = options.sessionId;
if (typeof sessionId === "string" ||
typeof sessionId === "number" ||
typeof sessionId === "boolean") {
metadata.sessionId = sessionId;
}
}
return {
isEnabled: true,
functionId,
metadata,
recordInputs: process.env.NEUROLINK_RECORD_INPUTS?.toLowerCase() !== "false",
recordOutputs: true,
};
}
/**
* Handle tool execution storage if available
*/
async handleToolExecutionStorage(toolCalls, toolResults, options, currentTime) {
// Check if tools are not empty
const hasToolData = (toolCalls && toolCalls.length > 0) ||
(toolResults && toolResults.length > 0);
// Check if NeuroLink instance is available and has tool execution storage
const hasStorageAvailable = this.neurolink?.isToolExecutionStorageAvailable();
// Early return if storage is not available or no tool data
if (!hasStorageAvailable || !hasToolData || !this.neurolink) {
return;
}
const sessionId = options.context?.sessionId ||
("sessionId" in options ? options.sessionId : undefined) ||
`session-${nanoid()}`;
const userId = options.context?.userId ||
("userId" in options ? options.userId : undefined);
try {
await this.neurolink.storeToolExecutions(sessionId, userId, toolCalls, toolResults, currentTime);
}
catch (error) {
logger.warn("Failed to store tool executions:", error);
}
}
}