spinal-obs-node
Version:
WithSpinal cost-aware OpenTelemetry SDK for Node.js
469 lines (467 loc) • 19.6 kB
JavaScript
import {
estimateCost
} from "./chunk-GCDHWKI2.js";
// src/analytics/index.ts
import { readFileSync } from "fs";
import { join } from "path";
var Analytics = class {
spans = [];
spansPath;
constructor(spansPath) {
this.spansPath = spansPath || join(process.cwd(), ".spinal", "spans.jsonl");
}
loadSpans() {
try {
const raw = readFileSync(this.spansPath, "utf8");
const lines = raw.trim().length ? raw.trim().split("\n") : [];
const spans = [];
for (const line of lines) {
try {
const span = JSON.parse(line);
spans.push(span);
} catch {
}
}
return spans;
} catch {
return [];
}
}
filterSpansByTime(spans, since) {
if (!since) return spans;
const now = Date.now();
const timeMap = {
"1h": 60 * 60 * 1e3,
"24h": 24 * 60 * 60 * 1e3,
"7d": 7 * 24 * 60 * 60 * 1e3,
"30d": 30 * 24 * 60 * 60 * 1e3,
"90d": 90 * 24 * 60 * 60 * 1e3,
"1y": 365 * 24 * 60 * 60 * 1e3
};
const cutoff = now - (timeMap[since] || 0);
return spans.filter((span) => {
const spanTime = span.start_time[0] * 1e3 + span.start_time[1] / 1e6;
return spanTime >= cutoff;
});
}
isOpenAISpan(span) {
return span.name === "openai-api-call" || span.attributes?.["spinal.provider"] === "openai" || span.instrumentation_info?.name === "spinal-openai";
}
getSpanDuration(span) {
const start = span.start_time[0] * 1e3 + span.start_time[1] / 1e6;
const end = span.end_time[0] * 1e3 + span.end_time[1] / 1e6;
return end - start;
}
analyzeCosts(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
let totalCost = 0;
const totalCalls = openAISpans.length;
const costByModel = {};
const costByAggregation = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const aggregationId = String(attrs["spinal.aggregation_id"] || "unknown");
const cost = estimateCost({ model, inputTokens, outputTokens });
totalCost += cost;
if (!costByModel[model]) {
costByModel[model] = { cost: 0, calls: 0, percentage: 0 };
}
costByModel[model].cost += cost;
costByModel[model].calls += 1;
if (!costByAggregation[aggregationId]) {
costByAggregation[aggregationId] = { cost: 0, calls: 0, percentage: 0 };
}
costByAggregation[aggregationId].cost += cost;
costByAggregation[aggregationId].calls += 1;
}
Object.values(costByModel).forEach((model) => {
model.percentage = totalCost > 0 ? model.cost / totalCost * 100 : 0;
});
Object.values(costByAggregation).forEach((agg) => {
agg.percentage = totalCost > 0 ? agg.cost / totalCost * 100 : 0;
});
const costTrends = this.calculateCostTrends(filteredSpans);
return {
totalCost,
totalCalls,
averageCostPerCall: totalCalls > 0 ? totalCost / totalCalls : 0,
costByModel,
costByAggregation,
costTrends
};
}
analyzeUsage(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const totalCalls = openAISpans.length;
let totalTokens = 0;
let inputTokens = 0;
let outputTokens = 0;
const usageByModel = {};
const usageByAggregation = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const spanInputTokens = Number(attrs["spinal.input_tokens"] || 0);
const spanOutputTokens = Number(attrs["spinal.output_tokens"] || 0);
const spanTotalTokens = Number(attrs["spinal.total_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const aggregationId = String(attrs["spinal.aggregation_id"] || "unknown");
inputTokens += spanInputTokens;
outputTokens += spanOutputTokens;
totalTokens += spanTotalTokens;
if (!usageByModel[model]) {
usageByModel[model] = { calls: 0, tokens: 0, percentage: 0 };
}
usageByModel[model].calls += 1;
usageByModel[model].tokens += spanTotalTokens;
if (!usageByAggregation[aggregationId]) {
usageByAggregation[aggregationId] = { calls: 0, tokens: 0, percentage: 0 };
}
usageByAggregation[aggregationId].calls += 1;
usageByAggregation[aggregationId].tokens += spanTotalTokens;
}
Object.values(usageByModel).forEach((model) => {
model.percentage = totalCalls > 0 ? model.calls / totalCalls * 100 : 0;
});
Object.values(usageByAggregation).forEach((agg) => {
agg.percentage = totalCalls > 0 ? agg.calls / totalCalls * 100 : 0;
});
return {
totalCalls,
totalTokens,
inputTokens,
outputTokens,
usageByModel,
usageByAggregation,
tokenEfficiency: {
averageInputTokensPerCall: totalCalls > 0 ? inputTokens / totalCalls : 0,
averageOutputTokensPerCall: totalCalls > 0 ? outputTokens / totalCalls : 0,
tokenRatio: inputTokens > 0 ? outputTokens / inputTokens : 0
}
};
}
analyzePerformance(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const totalRequests = openAISpans.length;
const successful = openAISpans.filter((span) => span.status.code === 1).length;
const failed = totalRequests - successful;
const successRate = totalRequests > 0 ? successful / totalRequests * 100 : 0;
const responseTimes = openAISpans.map((span) => this.getSpanDuration(span)).sort((a, b) => a - b);
const average = responseTimes.length > 0 ? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length : 0;
const median = responseTimes.length > 0 ? responseTimes[Math.floor(responseTimes.length / 2)] : 0;
const p95 = responseTimes.length > 0 ? responseTimes[Math.floor(responseTimes.length * 0.95)] : 0;
const fastest = responseTimes.length > 0 ? responseTimes[0] : 0;
const slowest = responseTimes.length > 0 ? responseTimes[responseTimes.length - 1] : 0;
const errors = {
rateLimit: 0,
authentication: 0,
network: 0,
other: failed
};
return {
totalRequests,
successful,
failed,
successRate,
responseTimes: {
average,
median,
p95,
fastest,
slowest
},
errors
};
}
analyzeModels(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const models = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const totalTokens = Number(attrs["spinal.total_tokens"] || 0);
const cost = estimateCost({ model, inputTokens, outputTokens });
const responseTime = this.getSpanDuration(span);
const isSuccess = span.status.code === 1;
if (!models[model]) {
models[model] = {
calls: 0,
totalCost: 0,
avgCostPerCall: 0,
avgResponseTime: 0,
successRate: 0,
totalTokens: 0
};
}
models[model].calls += 1;
models[model].totalCost += cost;
models[model].totalTokens += totalTokens;
models[model].avgResponseTime = (models[model].avgResponseTime * (models[model].calls - 1) + responseTime) / models[model].calls;
models[model].avgCostPerCall = models[model].totalCost / models[model].calls;
models[model].successRate = (models[model].successRate * (models[model].calls - 1) + (isSuccess ? 100 : 0)) / models[model].calls;
}
return { models };
}
analyzeAggregations(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const aggregations = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const aggregationId = String(attrs["spinal.aggregation_id"] || "unknown");
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const totalTokens = Number(attrs["spinal.total_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const cost = estimateCost({ model, inputTokens, outputTokens });
const isSuccess = span.status.code === 1;
if (!aggregations[aggregationId]) {
aggregations[aggregationId] = {
calls: 0,
totalCost: 0,
avgCostPerCall: 0,
successRate: 0,
totalTokens: 0
};
}
aggregations[aggregationId].calls += 1;
aggregations[aggregationId].totalCost += cost;
aggregations[aggregationId].totalTokens += totalTokens;
aggregations[aggregationId].avgCostPerCall = aggregations[aggregationId].totalCost / aggregations[aggregationId].calls;
aggregations[aggregationId].successRate = (aggregations[aggregationId].successRate * (aggregations[aggregationId].calls - 1) + (isSuccess ? 100 : 0)) / aggregations[aggregationId].calls;
}
return { aggregations };
}
analyzeTrends(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const dailyCalls = openAISpans.length / 30;
const peakUsage = { date: "unknown", calls: Math.max(...openAISpans.map(() => 1)) };
const growthRate = 0;
const totalCost = openAISpans.reduce((total, span) => {
const attrs = span.attributes || {};
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
return total + estimateCost({ model, inputTokens, outputTokens });
}, 0);
const dailyCost = totalCost / 30;
const peakCost = { date: "unknown", cost: totalCost };
const costPerCallTrend = "stable";
return {
usageTrends: {
dailyAverageCalls: dailyCalls,
peakUsage,
growthRate
},
costTrends: {
dailyAverageCost: dailyCost,
peakCost,
costPerCallTrend
},
performanceTrends: {
responseTimeTrend: "stable",
errorRateTrend: "stable",
successRateTrend: "stable"
}
};
}
getOptimizationRecommendations(options = {}) {
const costAnalysis = this.analyzeCosts(options);
const usageAnalysis = this.analyzeUsage(options);
const performanceAnalysis = this.analyzePerformance(options);
const modelAnalysis = this.analyzeModels(options);
const recommendations = {
costOptimization: [],
performanceOptimization: [],
usageOptimization: []
};
if (costAnalysis.averageCostPerCall > 0.02) {
recommendations.costOptimization.push("Consider using gpt-4o-mini for simple tasks to reduce costs");
}
if (costAnalysis.totalCost > 10) {
recommendations.costOptimization.push("Monitor token usage and optimize prompts to reduce costs");
}
if (performanceAnalysis.responseTimes.average > 3e3) {
recommendations.performanceOptimization.push("Consider implementing caching for repeated queries");
}
if (performanceAnalysis.successRate < 99) {
recommendations.performanceOptimization.push("Monitor error rates and implement retry logic");
}
const gpt4Usage = Object.entries(modelAnalysis.models).find(([model]) => model.includes("gpt-4o") && !model.includes("mini"));
if (gpt4Usage && gpt4Usage[1].calls > usageAnalysis.totalCalls * 0.5) {
recommendations.usageOptimization.push("Consider using gpt-4o-mini for simple tasks to reduce costs");
}
return recommendations;
}
analyzeResponses(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
let totalResponses = 0;
let totalResponseSize = 0;
const responseSizeDistribution = { small: 0, medium: 0, large: 0 };
const responseTypes = { success: 0, error: 0, truncated: 0 };
const errorTypes = {};
const errorMessages = [];
const modelResponseQuality = {};
const allResponseLengths = [];
for (const span of openAISpans) {
const attrs = span.attributes || {};
const responseData = attrs["spinal.response.binary_data"];
const responseSize = Number(attrs["spinal.response.size"] || 0);
const model = String(attrs["spinal.model"] || "unknown");
const isSuccess = span.status.code === 1;
if (responseData) {
totalResponses++;
totalResponseSize += responseSize;
if (responseSize < 500) responseSizeDistribution.small++;
else if (responseSize < 2e3) responseSizeDistribution.medium++;
else responseSizeDistribution.large++;
try {
const parsed = JSON.parse(responseData);
if (parsed.error) {
responseTypes.error++;
const errorType = parsed.error.type || "unknown";
errorTypes[errorType] = (errorTypes[errorType] || 0) + 1;
errorMessages.push(parsed.error.message || "Unknown error");
} else if (parsed.choices && parsed.choices.length > 0) {
responseTypes.success++;
const choice = parsed.choices[0];
const content = choice.message?.content || "";
const responseLength = content.length;
allResponseLengths.push(responseLength);
if (!modelResponseQuality[model]) {
modelResponseQuality[model] = {
averageResponseLength: 0,
averageResponseSize: 0,
successRate: 0,
commonErrors: [],
totalResponses: 0,
totalSize: 0,
totalLength: 0,
successful: 0
};
}
modelResponseQuality[model].totalResponses++;
modelResponseQuality[model].totalSize += responseSize;
modelResponseQuality[model].totalLength += responseLength;
modelResponseQuality[model].successful += isSuccess ? 1 : 0;
}
} catch {
responseTypes.truncated++;
}
}
}
const averageResponseSize = totalResponses > 0 ? totalResponseSize / totalResponses : 0;
const averageResponseLength = allResponseLengths.length > 0 ? allResponseLengths.reduce((a, b) => a + b, 0) / allResponseLengths.length : 0;
Object.keys(modelResponseQuality).forEach((model) => {
const data = modelResponseQuality[model];
data.averageResponseLength = data.totalResponses > 0 ? data.totalLength / data.totalResponses : 0;
data.averageResponseSize = data.totalResponses > 0 ? data.totalSize / data.totalResponses : 0;
data.successRate = data.totalResponses > 0 ? data.successful / data.totalResponses * 100 : 0;
});
return {
totalResponses,
averageResponseSize,
responseSizeDistribution,
contentPatterns: {
averageResponseLength,
commonPhrases: [],
// Could be enhanced with NLP analysis
responseTypes
},
errorAnalysis: {
totalErrors: responseTypes.error,
errorTypes,
errorMessages: [...new Set(errorMessages)],
// Remove duplicates
successRate: totalResponses > 0 ? responseTypes.success / totalResponses * 100 : 0
},
modelResponseQuality
};
}
getContentInsights(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const responsePatterns = { shortResponses: 0, mediumResponses: 0, longResponses: 0 };
const finishReasons = {};
const commonErrors = { rateLimit: 0, authentication: 0, modelNotFound: 0, other: 0 };
let totalOutputTokens = 0;
let totalResponseSize = 0;
let totalResponseLength = 0;
for (const span of openAISpans) {
const attrs = span.attributes || {};
const responseData = attrs["spinal.response.binary_data"];
const responseSize = Number(attrs["spinal.response.size"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
if (responseData) {
totalOutputTokens += outputTokens;
totalResponseSize += responseSize;
try {
const parsed = JSON.parse(responseData);
if (parsed.error) {
const errorType = parsed.error.type || "unknown";
if (errorType.includes("rate_limit")) commonErrors.rateLimit++;
else if (errorType.includes("auth")) commonErrors.authentication++;
else if (errorType.includes("model")) commonErrors.modelNotFound++;
else commonErrors.other++;
} else if (parsed.choices && parsed.choices.length > 0) {
const choice = parsed.choices[0];
const content = choice.message?.content || "";
const responseLength = content.length;
totalResponseLength += responseLength;
if (responseLength < 50) responsePatterns.shortResponses++;
else if (responseLength < 200) responsePatterns.mediumResponses++;
else responsePatterns.longResponses++;
const finishReason = choice.finish_reason || "unknown";
finishReasons[finishReason] = (finishReasons[finishReason] || 0) + 1;
}
} catch {
}
}
}
return {
responsePatterns,
finishReasons,
responseQuality: {
averageTokensPerCharacter: totalResponseLength > 0 ? totalOutputTokens / totalResponseLength : 0,
responseEfficiency: totalResponseSize > 0 ? totalOutputTokens / totalResponseSize : 0
},
commonErrors
};
}
calculateCostTrends(spans) {
const openAISpans = spans.filter((span) => this.isOpenAISpan(span));
if (openAISpans.length === 0) return [];
const totalCost = openAISpans.reduce((total, span) => {
const attrs = span.attributes || {};
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
return total + estimateCost({ model, inputTokens, outputTokens });
}, 0);
return [
{ date: "Today", cost: totalCost, calls: openAISpans.length }
];
}
};
export {
Analytics
};
//# sourceMappingURL=chunk-QPAMBULF.js.map