@juspay/neurolink
Version:
Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio
204 lines • 8.57 kB
JavaScript
/**
* Telemetry Handler Module
*
* Handles analytics, evaluation, performance metrics, and telemetry configuration.
* Extracted from BaseProvider to follow Single Responsibility Principle.
*
* Responsibilities:
* - Analytics creation and tracking
* - Evaluation generation
* - Performance metrics recording
* - Cost calculation
* - Telemetry configuration
*
* @module core/modules/TelemetryHandler
*/
import { nanoid } from "nanoid";
import { logger } from "../../utils/logger.js";
import { recordProviderPerformanceFromMetrics } from "../evaluationProviders.js";
import { modelConfig } from "../modelConfiguration.js";
import { TelemetryService } from "../../telemetry/telemetryService.js";
import { calculateCost, hasPricing } from "../../utils/pricing.js";
import { getLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
/**
* TelemetryHandler class - Handles analytics and telemetry for AI providers
*/
export class TelemetryHandler {
providerName;
modelName;
neurolink;
constructor(providerName, modelName, neurolink) {
this.providerName = providerName;
this.modelName = modelName;
this.neurolink = neurolink;
}
/**
* Create analytics for a generation result
*/
async createAnalytics(result, responseTime, context) {
const { createAnalytics } = await import("../analytics.js");
return createAnalytics(this.providerName, this.modelName, result, responseTime, context);
}
/**
* Create evaluation for a generation result
*/
async createEvaluation(result, options) {
const { evaluateResponse } = await import("../evaluation.js");
const context = {
userQuery: options.prompt || options.input?.text || "Generated response",
aiResponse: result.content,
context: options.context,
primaryDomain: options.evaluationDomain,
assistantRole: "AI assistant",
conversationHistory: options.conversationHistory?.map((msg) => ({
role: msg.role,
content: msg.content,
})),
toolUsage: options.toolUsageContext
? [
{
toolName: options.toolUsageContext,
input: {},
output: {},
executionTime: 0,
},
]
: undefined,
expectedOutcome: options.expectedOutcome,
evaluationCriteria: options.evaluationCriteria,
};
const evaluation = await evaluateResponse(context);
return evaluation;
}
/**
* Record performance metrics for a generation
*/
async recordPerformanceMetrics(usage, responseTime) {
try {
const totalTokens = (usage?.inputTokens || 0) + (usage?.outputTokens || 0);
const actualCost = await this.calculateActualCost(usage || { inputTokens: 0, outputTokens: 0 });
recordProviderPerformanceFromMetrics(this.providerName, {
responseTime,
tokensGenerated: totalTokens,
cost: actualCost,
success: true,
});
// Wire TelemetryService metrics so OTEL counters/histograms are populated
TelemetryService.getInstance().recordAIRequest(this.providerName, this.modelName, totalTokens, responseTime, actualCost > 0 ? actualCost : undefined);
logger.debug(`Performance recorded for ${this.providerName}`, {
responseTime: `${responseTime}ms`,
tokens: totalTokens,
estimatedCost: `$${actualCost.toFixed(6)}`,
});
}
catch (perfError) {
logger.warn("⚠️ Performance recording failed:", perfError);
}
}
/**
* Calculate actual cost based on token usage and provider configuration.
*
* Uses the per-model pricing table first (which has accurate rates for
* specific models like Claude on Vertex AI), then falls back to the
* provider-level default cost from modelConfiguration.
*
* Previously this only used modelConfig.getCostInfo() which returns
* provider-level defaults (e.g. Gemini rates for the "vertex" provider),
* causing a ~1,780x under-estimate when the actual model was Claude Sonnet
* on Vertex AI ($0.000060 vs $0.106895 for the same request).
*/
async calculateActualCost(usage) {
try {
const promptTokens = usage?.inputTokens || 0;
const completionTokens = usage?.outputTokens || 0;
// Try the per-model pricing table first (includes correct rates for
// Claude on Vertex, cache token rates, etc.)
if (hasPricing(this.providerName, this.modelName)) {
return calculateCost(this.providerName, this.modelName, {
input: promptTokens,
output: completionTokens,
total: promptTokens + completionTokens,
});
}
// Fall back to provider-level default cost from configuration system
const costInfo = modelConfig.getCostInfo(this.providerName, this.modelName);
if (!costInfo) {
return 0; // No cost info available
}
// Calculate cost per 1K tokens
const inputCost = (promptTokens / 1000) * costInfo.input;
const outputCost = (completionTokens / 1000) * costInfo.output;
return inputCost + outputCost;
}
catch (error) {
logger.debug(`Cost calculation failed for ${this.providerName}:`, error);
return 0; // Fallback to 0 on any error
}
}
/**
* Create telemetry configuration for Vercel AI SDK experimental_telemetry
* This enables automatic OpenTelemetry tracing when telemetry is enabled
*/
getTelemetryConfig(options, operationType = "stream") {
// Check if telemetry is enabled via NeuroLink observability config
if (!this.neurolink?.isTelemetryEnabled()) {
return undefined;
}
const context = options.context;
const langfuseContext = getLangfuseContext();
const traceName = context?.traceName ?? langfuseContext?.traceName;
const userId = context?.userId ?? langfuseContext?.userId;
const functionId = traceName ? traceName : userId ? userId : "guest";
const metadata = {
...(context?.metadata || {}),
provider: this.providerName,
model: this.modelName,
toolsEnabled: !options.disableTools,
neurolink: true,
operationType,
originalProvider: this.providerName,
};
// Add sessionId if available
if ("sessionId" in options && options.sessionId) {
const sessionId = options.sessionId;
if (typeof sessionId === "string" ||
typeof sessionId === "number" ||
typeof sessionId === "boolean") {
metadata.sessionId = sessionId;
}
}
return {
isEnabled: true,
functionId,
metadata,
recordInputs: process.env.NEUROLINK_RECORD_INPUTS?.toLowerCase() !== "false",
recordOutputs: true,
};
}
/**
* Handle tool execution storage if available
*/
async handleToolExecutionStorage(toolCalls, toolResults, options, currentTime) {
// Check if tools are not empty
const hasToolData = (toolCalls && toolCalls.length > 0) ||
(toolResults && toolResults.length > 0);
// Check if NeuroLink instance is available and has tool execution storage
const hasStorageAvailable = this.neurolink?.isToolExecutionStorageAvailable();
// Early return if storage is not available or no tool data
if (!hasStorageAvailable || !hasToolData || !this.neurolink) {
return;
}
const sessionId = options.context?.sessionId ||
options.sessionId ||
`session-${nanoid()}`;
const userId = options.context?.userId ||
options.userId;
try {
await this.neurolink.storeToolExecutions(sessionId, userId, toolCalls, toolResults, currentTime);
}
catch (error) {
logger.warn("Failed to store tool executions:", error);
}
}
}
//# sourceMappingURL=TelemetryHandler.js.map