spinal-obs-node
Version:
WithSpinal cost-aware OpenTelemetry SDK for Node.js
1,011 lines (1,003 loc) • 55.9 kB
JavaScript
#!/usr/bin/env node
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
// src/cli/index.ts
var import_commander2 = require("commander");
var import_conf = __toESM(require("conf"), 1);
var import_open = __toESM(require("open"), 1);
// src/runtime/config.ts
var import_api = require("@opentelemetry/api");
var import_path = __toESM(require("path"), 1);
var DefaultScrubber = class {
sensitive = [
/password/i,
/passwd/i,
/secret/i,
/api[._-]?key/i,
/auth[._-]?token/i,
/access[._-]?token/i,
/private[._-]?key/i,
/encryption[._-]?key/i,
/bearer/i,
/credential/i,
/user[._-]?name/i,
/first[._-]?name/i,
/last[._-]?name/i,
/email/i,
/email[._-]?address/i,
/phone[._-]?number/i,
/ip[._-]?address/i
];
protected = [/^attributes$/i, /^spinal\./i];
scrubAttributes(attributes) {
const out = {};
for (const [k, v] of Object.entries(attributes ?? {})) {
if (this.sensitive.some((r) => r.test(k))) {
out[k] = "[Scrubbed]";
} else if (Array.isArray(v)) {
out[k] = v.map((x) => typeof x === "object" && x !== null ? this.scrubAttributes(x) : x);
} else if (typeof v === "object" && v !== null) {
out[k] = this.scrubAttributes(v);
} else {
out[k] = v;
}
}
return out;
}
};
var globalConfig;
function configure(opts = {}) {
const endpoint = opts.endpoint ?? process.env.SPINAL_TRACING_ENDPOINT ?? "https://cloud.withspinal.com";
const apiKey = opts.apiKey ?? process.env.SPINAL_API_KEY ?? "";
const disableLocalMode = opts.disableLocalMode ?? process.env.SPINAL_DISABLE_LOCAL_MODE === "true";
const inferredMode = (opts.mode ?? process.env.SPINAL_MODE) || (apiKey ? "cloud" : "local");
const mode = disableLocalMode && !apiKey ? (() => {
throw new Error("Cannot disable local mode without providing an API key for cloud mode");
})() : inferredMode;
const headers = mode === "cloud" ? { ...opts.headers ?? {}, "X-SPINAL-API-KEY": apiKey } : { ...opts.headers ?? {} };
const timeoutMs = opts.timeoutMs ?? 5e3;
const maxQueueSize = opts.maxQueueSize ?? parseInt(process.env.SPINAL_PROCESS_MAX_QUEUE_SIZE ?? "2048", 10);
const maxExportBatchSize = opts.maxExportBatchSize ?? parseInt(process.env.SPINAL_PROCESS_MAX_EXPORT_BATCH_SIZE ?? "512", 10);
const scheduleDelayMs = opts.scheduleDelayMs ?? parseInt(process.env.SPINAL_PROCESS_SCHEDULE_DELAY ?? "5000", 10);
const exportTimeoutMs = opts.exportTimeoutMs ?? parseInt(process.env.SPINAL_PROCESS_EXPORT_TIMEOUT ?? "30000", 10);
const scrubber = opts.scrubber ?? new DefaultScrubber();
const opentelemetryLogLevel = opts.opentelemetryLogLevel ?? import_api.DiagLogLevel.ERROR;
const localStorePath = opts.localStorePath ?? process.env.SPINAL_LOCAL_STORE_PATH ?? import_path.default.join(process.cwd(), ".spinal", "spans.jsonl");
if (!endpoint) throw new Error("Spinal endpoint must be provided");
if (mode === "cloud" && !apiKey) throw new Error("No API key provided. Set SPINAL_API_KEY or pass to configure().");
import_api.diag.setLogger(console, opentelemetryLogLevel);
globalConfig = {
endpoint,
apiKey,
headers,
timeoutMs,
maxQueueSize,
maxExportBatchSize,
scheduleDelayMs,
exportTimeoutMs,
scrubber,
opentelemetryLogLevel,
mode,
localStorePath,
disableLocalMode
};
return globalConfig;
}
function getConfig() {
if (!globalConfig) return configure();
return globalConfig;
}
// src/cli/index.ts
var import_fs2 = __toESM(require("fs"), 1);
// src/pricing/index.ts
var catalog = [
{ model: "openai:gpt-4o-mini", inputPer1K: 0.15, outputPer1K: 0.6 },
{ model: "openai:gpt-4o", inputPer1K: 2.5, outputPer1K: 10 }
];
function estimateCost(params) {
const { model = "openai:gpt-4o-mini", inputTokens = 0, outputTokens = 0 } = params;
const entry = catalog.find((c) => c.model === model) ?? catalog[0];
const inputCost = inputTokens / 1e3 * entry.inputPer1K;
const outputCost = outputTokens / 1e3 * entry.outputPer1K;
return roundUSD(inputCost + outputCost);
}
function roundUSD(n) {
return Math.round(n * 1e4) / 1e4;
}
// src/cli/analytics.ts
var import_commander = require("commander");
// src/analytics/index.ts
var import_fs = require("fs");
var import_path2 = require("path");
var Analytics = class {
spans = [];
spansPath;
constructor(spansPath) {
this.spansPath = spansPath || (0, import_path2.join)(process.cwd(), ".spinal", "spans.jsonl");
}
loadSpans() {
try {
const raw = (0, import_fs.readFileSync)(this.spansPath, "utf8");
const lines = raw.trim().length ? raw.trim().split("\n") : [];
const spans = [];
for (const line of lines) {
try {
const span = JSON.parse(line);
spans.push(span);
} catch {
}
}
return spans;
} catch {
return [];
}
}
filterSpansByTime(spans, since) {
if (!since) return spans;
const now = Date.now();
const timeMap = {
"1h": 60 * 60 * 1e3,
"24h": 24 * 60 * 60 * 1e3,
"7d": 7 * 24 * 60 * 60 * 1e3,
"30d": 30 * 24 * 60 * 60 * 1e3,
"90d": 90 * 24 * 60 * 60 * 1e3,
"1y": 365 * 24 * 60 * 60 * 1e3
};
const cutoff = now - (timeMap[since] || 0);
return spans.filter((span) => {
const spanTime = span.start_time[0] * 1e3 + span.start_time[1] / 1e6;
return spanTime >= cutoff;
});
}
isOpenAISpan(span) {
return span.name === "openai-api-call" || span.attributes?.["spinal.provider"] === "openai" || span.instrumentation_info?.name === "spinal-openai";
}
getSpanDuration(span) {
const start = span.start_time[0] * 1e3 + span.start_time[1] / 1e6;
const end = span.end_time[0] * 1e3 + span.end_time[1] / 1e6;
return end - start;
}
analyzeCosts(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
let totalCost = 0;
const totalCalls = openAISpans.length;
const costByModel = {};
const costByAggregation = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const aggregationId = String(attrs["spinal.aggregation_id"] || "unknown");
const cost = estimateCost({ model, inputTokens, outputTokens });
totalCost += cost;
if (!costByModel[model]) {
costByModel[model] = { cost: 0, calls: 0, percentage: 0 };
}
costByModel[model].cost += cost;
costByModel[model].calls += 1;
if (!costByAggregation[aggregationId]) {
costByAggregation[aggregationId] = { cost: 0, calls: 0, percentage: 0 };
}
costByAggregation[aggregationId].cost += cost;
costByAggregation[aggregationId].calls += 1;
}
Object.values(costByModel).forEach((model) => {
model.percentage = totalCost > 0 ? model.cost / totalCost * 100 : 0;
});
Object.values(costByAggregation).forEach((agg) => {
agg.percentage = totalCost > 0 ? agg.cost / totalCost * 100 : 0;
});
const costTrends = this.calculateCostTrends(filteredSpans);
return {
totalCost,
totalCalls,
averageCostPerCall: totalCalls > 0 ? totalCost / totalCalls : 0,
costByModel,
costByAggregation,
costTrends
};
}
analyzeUsage(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const totalCalls = openAISpans.length;
let totalTokens = 0;
let inputTokens = 0;
let outputTokens = 0;
const usageByModel = {};
const usageByAggregation = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const spanInputTokens = Number(attrs["spinal.input_tokens"] || 0);
const spanOutputTokens = Number(attrs["spinal.output_tokens"] || 0);
const spanTotalTokens = Number(attrs["spinal.total_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const aggregationId = String(attrs["spinal.aggregation_id"] || "unknown");
inputTokens += spanInputTokens;
outputTokens += spanOutputTokens;
totalTokens += spanTotalTokens;
if (!usageByModel[model]) {
usageByModel[model] = { calls: 0, tokens: 0, percentage: 0 };
}
usageByModel[model].calls += 1;
usageByModel[model].tokens += spanTotalTokens;
if (!usageByAggregation[aggregationId]) {
usageByAggregation[aggregationId] = { calls: 0, tokens: 0, percentage: 0 };
}
usageByAggregation[aggregationId].calls += 1;
usageByAggregation[aggregationId].tokens += spanTotalTokens;
}
Object.values(usageByModel).forEach((model) => {
model.percentage = totalCalls > 0 ? model.calls / totalCalls * 100 : 0;
});
Object.values(usageByAggregation).forEach((agg) => {
agg.percentage = totalCalls > 0 ? agg.calls / totalCalls * 100 : 0;
});
return {
totalCalls,
totalTokens,
inputTokens,
outputTokens,
usageByModel,
usageByAggregation,
tokenEfficiency: {
averageInputTokensPerCall: totalCalls > 0 ? inputTokens / totalCalls : 0,
averageOutputTokensPerCall: totalCalls > 0 ? outputTokens / totalCalls : 0,
tokenRatio: inputTokens > 0 ? outputTokens / inputTokens : 0
}
};
}
analyzePerformance(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const totalRequests = openAISpans.length;
const successful = openAISpans.filter((span) => span.status.code === 1).length;
const failed = totalRequests - successful;
const successRate = totalRequests > 0 ? successful / totalRequests * 100 : 0;
const responseTimes = openAISpans.map((span) => this.getSpanDuration(span)).sort((a, b) => a - b);
const average = responseTimes.length > 0 ? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length : 0;
const median = responseTimes.length > 0 ? responseTimes[Math.floor(responseTimes.length / 2)] : 0;
const p95 = responseTimes.length > 0 ? responseTimes[Math.floor(responseTimes.length * 0.95)] : 0;
const fastest = responseTimes.length > 0 ? responseTimes[0] : 0;
const slowest = responseTimes.length > 0 ? responseTimes[responseTimes.length - 1] : 0;
const errors = {
rateLimit: 0,
authentication: 0,
network: 0,
other: failed
};
return {
totalRequests,
successful,
failed,
successRate,
responseTimes: {
average,
median,
p95,
fastest,
slowest
},
errors
};
}
analyzeModels(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const models = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const totalTokens = Number(attrs["spinal.total_tokens"] || 0);
const cost = estimateCost({ model, inputTokens, outputTokens });
const responseTime = this.getSpanDuration(span);
const isSuccess = span.status.code === 1;
if (!models[model]) {
models[model] = {
calls: 0,
totalCost: 0,
avgCostPerCall: 0,
avgResponseTime: 0,
successRate: 0,
totalTokens: 0
};
}
models[model].calls += 1;
models[model].totalCost += cost;
models[model].totalTokens += totalTokens;
models[model].avgResponseTime = (models[model].avgResponseTime * (models[model].calls - 1) + responseTime) / models[model].calls;
models[model].avgCostPerCall = models[model].totalCost / models[model].calls;
models[model].successRate = (models[model].successRate * (models[model].calls - 1) + (isSuccess ? 100 : 0)) / models[model].calls;
}
return { models };
}
analyzeAggregations(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const aggregations = {};
for (const span of openAISpans) {
const attrs = span.attributes || {};
const aggregationId = String(attrs["spinal.aggregation_id"] || "unknown");
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const totalTokens = Number(attrs["spinal.total_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
const cost = estimateCost({ model, inputTokens, outputTokens });
const isSuccess = span.status.code === 1;
if (!aggregations[aggregationId]) {
aggregations[aggregationId] = {
calls: 0,
totalCost: 0,
avgCostPerCall: 0,
successRate: 0,
totalTokens: 0
};
}
aggregations[aggregationId].calls += 1;
aggregations[aggregationId].totalCost += cost;
aggregations[aggregationId].totalTokens += totalTokens;
aggregations[aggregationId].avgCostPerCall = aggregations[aggregationId].totalCost / aggregations[aggregationId].calls;
aggregations[aggregationId].successRate = (aggregations[aggregationId].successRate * (aggregations[aggregationId].calls - 1) + (isSuccess ? 100 : 0)) / aggregations[aggregationId].calls;
}
return { aggregations };
}
analyzeTrends(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const dailyCalls = openAISpans.length / 30;
const peakUsage = { date: "unknown", calls: Math.max(...openAISpans.map(() => 1)) };
const growthRate = 0;
const totalCost = openAISpans.reduce((total, span) => {
const attrs = span.attributes || {};
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
return total + estimateCost({ model, inputTokens, outputTokens });
}, 0);
const dailyCost = totalCost / 30;
const peakCost = { date: "unknown", cost: totalCost };
const costPerCallTrend = "stable";
return {
usageTrends: {
dailyAverageCalls: dailyCalls,
peakUsage,
growthRate
},
costTrends: {
dailyAverageCost: dailyCost,
peakCost,
costPerCallTrend
},
performanceTrends: {
responseTimeTrend: "stable",
errorRateTrend: "stable",
successRateTrend: "stable"
}
};
}
getOptimizationRecommendations(options = {}) {
const costAnalysis = this.analyzeCosts(options);
const usageAnalysis = this.analyzeUsage(options);
const performanceAnalysis = this.analyzePerformance(options);
const modelAnalysis = this.analyzeModels(options);
const recommendations = {
costOptimization: [],
performanceOptimization: [],
usageOptimization: []
};
if (costAnalysis.averageCostPerCall > 0.02) {
recommendations.costOptimization.push("Consider using gpt-4o-mini for simple tasks to reduce costs");
}
if (costAnalysis.totalCost > 10) {
recommendations.costOptimization.push("Monitor token usage and optimize prompts to reduce costs");
}
if (performanceAnalysis.responseTimes.average > 3e3) {
recommendations.performanceOptimization.push("Consider implementing caching for repeated queries");
}
if (performanceAnalysis.successRate < 99) {
recommendations.performanceOptimization.push("Monitor error rates and implement retry logic");
}
const gpt4Usage = Object.entries(modelAnalysis.models).find(([model]) => model.includes("gpt-4o") && !model.includes("mini"));
if (gpt4Usage && gpt4Usage[1].calls > usageAnalysis.totalCalls * 0.5) {
recommendations.usageOptimization.push("Consider using gpt-4o-mini for simple tasks to reduce costs");
}
return recommendations;
}
analyzeResponses(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
let totalResponses = 0;
let totalResponseSize = 0;
const responseSizeDistribution = { small: 0, medium: 0, large: 0 };
const responseTypes = { success: 0, error: 0, truncated: 0 };
const errorTypes = {};
const errorMessages = [];
const modelResponseQuality = {};
const allResponseLengths = [];
for (const span of openAISpans) {
const attrs = span.attributes || {};
const responseData = attrs["spinal.response.binary_data"];
const responseSize = Number(attrs["spinal.response.size"] || 0);
const model = String(attrs["spinal.model"] || "unknown");
const isSuccess = span.status.code === 1;
if (responseData) {
totalResponses++;
totalResponseSize += responseSize;
if (responseSize < 500) responseSizeDistribution.small++;
else if (responseSize < 2e3) responseSizeDistribution.medium++;
else responseSizeDistribution.large++;
try {
const parsed = JSON.parse(responseData);
if (parsed.error) {
responseTypes.error++;
const errorType = parsed.error.type || "unknown";
errorTypes[errorType] = (errorTypes[errorType] || 0) + 1;
errorMessages.push(parsed.error.message || "Unknown error");
} else if (parsed.choices && parsed.choices.length > 0) {
responseTypes.success++;
const choice = parsed.choices[0];
const content = choice.message?.content || "";
const responseLength = content.length;
allResponseLengths.push(responseLength);
if (!modelResponseQuality[model]) {
modelResponseQuality[model] = {
averageResponseLength: 0,
averageResponseSize: 0,
successRate: 0,
commonErrors: [],
totalResponses: 0,
totalSize: 0,
totalLength: 0,
successful: 0
};
}
modelResponseQuality[model].totalResponses++;
modelResponseQuality[model].totalSize += responseSize;
modelResponseQuality[model].totalLength += responseLength;
modelResponseQuality[model].successful += isSuccess ? 1 : 0;
}
} catch {
responseTypes.truncated++;
}
}
}
const averageResponseSize = totalResponses > 0 ? totalResponseSize / totalResponses : 0;
const averageResponseLength = allResponseLengths.length > 0 ? allResponseLengths.reduce((a, b) => a + b, 0) / allResponseLengths.length : 0;
Object.keys(modelResponseQuality).forEach((model) => {
const data = modelResponseQuality[model];
data.averageResponseLength = data.totalResponses > 0 ? data.totalLength / data.totalResponses : 0;
data.averageResponseSize = data.totalResponses > 0 ? data.totalSize / data.totalResponses : 0;
data.successRate = data.totalResponses > 0 ? data.successful / data.totalResponses * 100 : 0;
});
return {
totalResponses,
averageResponseSize,
responseSizeDistribution,
contentPatterns: {
averageResponseLength,
commonPhrases: [],
// Could be enhanced with NLP analysis
responseTypes
},
errorAnalysis: {
totalErrors: responseTypes.error,
errorTypes,
errorMessages: [...new Set(errorMessages)],
// Remove duplicates
successRate: totalResponses > 0 ? responseTypes.success / totalResponses * 100 : 0
},
modelResponseQuality
};
}
getContentInsights(options = {}) {
this.spans = this.loadSpans();
const filteredSpans = this.filterSpansByTime(this.spans, options.since);
const openAISpans = filteredSpans.filter((span) => this.isOpenAISpan(span));
const responsePatterns = { shortResponses: 0, mediumResponses: 0, longResponses: 0 };
const finishReasons = {};
const commonErrors = { rateLimit: 0, authentication: 0, modelNotFound: 0, other: 0 };
let totalOutputTokens = 0;
let totalResponseSize = 0;
let totalResponseLength = 0;
for (const span of openAISpans) {
const attrs = span.attributes || {};
const responseData = attrs["spinal.response.binary_data"];
const responseSize = Number(attrs["spinal.response.size"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
if (responseData) {
totalOutputTokens += outputTokens;
totalResponseSize += responseSize;
try {
const parsed = JSON.parse(responseData);
if (parsed.error) {
const errorType = parsed.error.type || "unknown";
if (errorType.includes("rate_limit")) commonErrors.rateLimit++;
else if (errorType.includes("auth")) commonErrors.authentication++;
else if (errorType.includes("model")) commonErrors.modelNotFound++;
else commonErrors.other++;
} else if (parsed.choices && parsed.choices.length > 0) {
const choice = parsed.choices[0];
const content = choice.message?.content || "";
const responseLength = content.length;
totalResponseLength += responseLength;
if (responseLength < 50) responsePatterns.shortResponses++;
else if (responseLength < 200) responsePatterns.mediumResponses++;
else responsePatterns.longResponses++;
const finishReason = choice.finish_reason || "unknown";
finishReasons[finishReason] = (finishReasons[finishReason] || 0) + 1;
}
} catch {
}
}
}
return {
responsePatterns,
finishReasons,
responseQuality: {
averageTokensPerCharacter: totalResponseLength > 0 ? totalOutputTokens / totalResponseLength : 0,
responseEfficiency: totalResponseSize > 0 ? totalOutputTokens / totalResponseSize : 0
},
commonErrors
};
}
calculateCostTrends(spans) {
const openAISpans = spans.filter((span) => this.isOpenAISpan(span));
if (openAISpans.length === 0) return [];
const totalCost = openAISpans.reduce((total, span) => {
const attrs = span.attributes || {};
const inputTokens = Number(attrs["spinal.input_tokens"] || 0);
const outputTokens = Number(attrs["spinal.output_tokens"] || 0);
const model = String(attrs["spinal.model"] || "openai:gpt-4o-mini");
return total + estimateCost({ model, inputTokens, outputTokens });
}, 0);
return [
{ date: "Today", cost: totalCost, calls: openAISpans.length }
];
}
};
// src/cli/analytics.ts
function createAnalyticsCommands() {
const commands = [];
const costCommand = new import_commander.Command("cost").description("Analyze OpenAI API costs").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "7d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--by-model", "Show breakdown by model").option("--by-aggregation", "Show breakdown by aggregation ID").option("--trends", "Show cost trends").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzeCosts({
since: options.since,
format: options.format,
byModel: options.byModel,
byAggregation: options.byAggregation,
trends: options.trends
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "csv") {
console.log("Period,Total Cost,Total Calls,Average Cost Per Call");
console.log(`${options.since},${analysis.totalCost.toFixed(6)},${analysis.totalCalls},${analysis.averageCostPerCall.toFixed(6)}`);
} else if (options.format === "summary") {
console.log(`\u{1F4B0} Total Cost: $${analysis.totalCost.toFixed(4)}`);
console.log(`\u{1F4CA} Total Calls: ${analysis.totalCalls}`);
console.log(`\u{1F4C8} Average Cost per Call: $${analysis.averageCostPerCall.toFixed(6)}`);
} else {
console.log(`
\u{1F4B0} Cost Analysis (Last ${options.since})
`);
console.log("\u2500".repeat(60));
console.log(`Total Cost: $${analysis.totalCost.toFixed(4)}`);
console.log(`Total API Calls: ${analysis.totalCalls}`);
console.log(`Average Cost per Call: $${analysis.averageCostPerCall.toFixed(6)}`);
console.log("\u2500".repeat(60));
if (options.byModel && Object.keys(analysis.costByModel).length > 0) {
console.log("\n\u{1F4CA} Cost by Model:");
Object.entries(analysis.costByModel).forEach(([model, data]) => {
console.log(`\u2022 ${model}: $${data.cost.toFixed(4)} (${data.calls} calls, ${data.percentage.toFixed(1)}%)`);
});
}
if (options.byAggregation && Object.keys(analysis.costByAggregation).length > 0) {
console.log("\n\u{1F3F7}\uFE0F Cost by Aggregation:");
Object.entries(analysis.costByAggregation).forEach(([agg, data]) => {
console.log(`\u2022 ${agg}: $${data.cost.toFixed(4)} (${data.calls} calls, ${data.percentage.toFixed(1)}%)`);
});
}
if (options.trends && analysis.costTrends.length > 0) {
console.log("\n\u{1F4C8} Cost Trends:");
analysis.costTrends.forEach((trend) => {
console.log(`\u2022 ${trend.date}: $${trend.cost.toFixed(4)} (${trend.calls} calls)`);
});
}
}
});
const usageCommand = new import_commander.Command("usage").description("Analyze OpenAI API usage patterns").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "24h").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--tokens", "Show token breakdown").option("--by-model", "Show usage by model").option("--by-aggregation", "Show usage by aggregation ID").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzeUsage({
since: options.since,
format: options.format,
byModel: options.byModel,
byAggregation: options.byAggregation
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "csv") {
console.log("Period,Total Calls,Total Tokens,Input Tokens,Output Tokens");
console.log(`${options.since},${analysis.totalCalls},${analysis.totalTokens},${analysis.inputTokens},${analysis.outputTokens}`);
} else if (options.format === "summary") {
console.log(`\u{1F4CA} Total Calls: ${analysis.totalCalls}`);
console.log(`\u{1F524} Total Tokens: ${analysis.totalTokens.toLocaleString()}`);
console.log(`\u{1F4E5} Input Tokens: ${analysis.inputTokens.toLocaleString()}`);
console.log(`\u{1F4E4} Output Tokens: ${analysis.outputTokens.toLocaleString()}`);
} else {
console.log(`
\u{1F4CA} Usage Analytics (Last ${options.since})
`);
console.log("\u2500".repeat(60));
console.log(`Total API Calls: ${analysis.totalCalls}`);
console.log(`Total Tokens: ${analysis.totalTokens.toLocaleString()}`);
console.log(`\u2022 Input tokens: ${analysis.inputTokens.toLocaleString()}`);
console.log(`\u2022 Output tokens: ${analysis.outputTokens.toLocaleString()}`);
console.log("\u2500".repeat(60));
if (options.tokens) {
console.log("\n\u{1F4C8} Token Efficiency:");
console.log(`\u2022 Average input tokens per call: ${analysis.tokenEfficiency.averageInputTokensPerCall.toFixed(1)}`);
console.log(`\u2022 Average output tokens per call: ${analysis.tokenEfficiency.averageOutputTokensPerCall.toFixed(1)}`);
console.log(`\u2022 Token ratio (output/input): ${analysis.tokenEfficiency.tokenRatio.toFixed(2)}`);
}
if (options.byModel && Object.keys(analysis.usageByModel).length > 0) {
console.log("\n\u{1F916} Usage by Model:");
Object.entries(analysis.usageByModel).forEach(([model, data]) => {
console.log(`\u2022 ${model}: ${data.calls} calls (${data.percentage.toFixed(1)}%), ${data.tokens.toLocaleString()} tokens`);
});
}
if (options.byAggregation && Object.keys(analysis.usageByAggregation).length > 0) {
console.log("\n\u{1F3F7}\uFE0F Usage by Aggregation:");
Object.entries(analysis.usageByAggregation).forEach(([agg, data]) => {
console.log(`\u2022 ${agg}: ${data.calls} calls (${data.percentage.toFixed(1)}%), ${data.tokens.toLocaleString()} tokens`);
});
}
}
});
const performanceCommand = new import_commander.Command("performance").description("Analyze OpenAI API performance").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "7d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--response-times", "Show response time analysis").option("--errors", "Show error analysis").option("--by-model", "Show performance by model").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzePerformance({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "csv") {
console.log("Period,Total Requests,Successful,Failed,Success Rate,Avg Response Time");
console.log(`${options.since},${analysis.totalRequests},${analysis.successful},${analysis.failed},${analysis.successRate.toFixed(1)}%,${analysis.responseTimes.average.toFixed(1)}ms`);
} else if (options.format === "summary") {
console.log(`\u26A1 Total Requests: ${analysis.totalRequests}`);
console.log(`\u2705 Successful: ${analysis.successful} (${analysis.successRate.toFixed(1)}%)`);
console.log(`\u274C Failed: ${analysis.failed}`);
console.log(`\u23F1\uFE0F Avg Response Time: ${analysis.responseTimes.average.toFixed(1)}ms`);
} else {
console.log(`
\u26A1 Performance Analytics (Last ${options.since})
`);
console.log("\u2500".repeat(60));
console.log(`Total Requests: ${analysis.totalRequests}`);
console.log(`Successful: ${analysis.successful} (${analysis.successRate.toFixed(1)}%)`);
console.log(`Failed: ${analysis.failed} (${(100 - analysis.successRate).toFixed(1)}%)`);
console.log("\u2500".repeat(60));
if (options.responseTimes) {
console.log("\n\u23F1\uFE0F Response Times:");
console.log(`\u2022 Average: ${analysis.responseTimes.average.toFixed(1)}ms`);
console.log(`\u2022 Median: ${analysis.responseTimes.median.toFixed(1)}ms`);
console.log(`\u2022 95th percentile: ${analysis.responseTimes.p95.toFixed(1)}ms`);
console.log(`\u2022 Fastest: ${analysis.responseTimes.fastest.toFixed(1)}ms`);
console.log(`\u2022 Slowest: ${analysis.responseTimes.slowest.toFixed(1)}ms`);
}
if (options.errors && analysis.errors.other > 0) {
console.log("\n\u{1F6A8} Error Analysis:");
console.log(`\u2022 Rate limit errors: ${analysis.errors.rateLimit}`);
console.log(`\u2022 Authentication errors: ${analysis.errors.authentication}`);
console.log(`\u2022 Network errors: ${analysis.errors.network}`);
console.log(`\u2022 Other errors: ${analysis.errors.other}`);
}
}
});
const modelsCommand = new import_commander.Command("models").description("Compare performance and costs across different models").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "30d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--efficiency", "Show model efficiency metrics").option("--costs", "Show cost analysis by model").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzeModels({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "summary") {
Object.entries(analysis.models).forEach(([model, data]) => {
console.log(`${model}: ${data.calls} calls, $${data.totalCost.toFixed(4)}, ${data.successRate.toFixed(1)}% success`);
});
} else {
console.log(`
\u{1F916} Model Analytics (Last ${options.since})
`);
console.log("\u2500".repeat(80));
Object.entries(analysis.models).forEach(([model, data]) => {
console.log(`${model}:`);
console.log(` \u2022 Calls: ${data.calls}`);
console.log(` \u2022 Total cost: $${data.totalCost.toFixed(4)}`);
console.log(` \u2022 Avg cost per call: $${data.avgCostPerCall.toFixed(6)}`);
console.log(` \u2022 Avg response time: ${data.avgResponseTime.toFixed(1)}ms`);
console.log(` \u2022 Success rate: ${data.successRate.toFixed(1)}%`);
console.log(` \u2022 Total tokens: ${data.totalTokens.toLocaleString()}`);
console.log("");
});
}
});
const aggregationsCommand = new import_commander.Command("aggregations").description("Analyze usage by custom aggregation IDs").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "7d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--id <aggregationId>", "Show specific aggregation ID").option("--costs", "Show cost analysis by aggregation").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzeAggregations({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "summary") {
Object.entries(analysis.aggregations).forEach(([agg, data]) => {
console.log(`${agg}: ${data.calls} calls, $${data.totalCost.toFixed(4)}, ${data.successRate.toFixed(1)}% success`);
});
} else {
console.log(`
\u{1F3F7}\uFE0F Aggregation Analytics (Last ${options.since})
`);
console.log("\u2500".repeat(80));
Object.entries(analysis.aggregations).forEach(([agg, data]) => {
console.log(`${agg}:`);
console.log(` \u2022 Calls: ${data.calls}`);
console.log(` \u2022 Total cost: $${data.totalCost.toFixed(4)}`);
console.log(` \u2022 Avg cost per call: $${data.avgCostPerCall.toFixed(6)}`);
console.log(` \u2022 Success rate: ${data.successRate.toFixed(1)}%`);
console.log(` \u2022 Total tokens: ${data.totalTokens.toLocaleString()}`);
console.log("");
});
}
});
const trendsCommand = new import_commander.Command("trends").description("Identify usage patterns and trends over time").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "30d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--costs", "Show cost trends").option("--usage", "Show usage trends").option("--performance", "Show performance trends").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzeTrends({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "summary") {
console.log(`\u{1F4CA} Daily avg calls: ${analysis.usageTrends.dailyAverageCalls.toFixed(1)}`);
console.log(`\u{1F4B0} Daily avg cost: $${analysis.costTrends.dailyAverageCost.toFixed(4)}`);
console.log(`\u26A1 Performance: ${analysis.performanceTrends.responseTimeTrend}`);
} else {
console.log(`
\u{1F4C8} Trends Analysis (Last ${options.since})
`);
console.log("\u2500".repeat(60));
if (options.usage || !options.costs && !options.performance) {
console.log("\n\u{1F4CA} Usage Trends:");
console.log(`\u2022 Daily average calls: ${analysis.usageTrends.dailyAverageCalls.toFixed(1)}`);
console.log(`\u2022 Peak usage: ${analysis.usageTrends.peakUsage.calls} calls (${analysis.usageTrends.peakUsage.date})`);
console.log(`\u2022 Growth rate: ${analysis.usageTrends.growthRate > 0 ? "+" : ""}${analysis.usageTrends.growthRate.toFixed(1)}% week-over-week`);
}
if (options.costs || !options.usage && !options.performance) {
console.log("\n\u{1F4B0} Cost Trends:");
console.log(`\u2022 Daily average cost: $${analysis.costTrends.dailyAverageCost.toFixed(4)}`);
console.log(`\u2022 Peak cost: $${analysis.costTrends.peakCost.cost.toFixed(4)} (${analysis.costTrends.peakCost.date})`);
console.log(`\u2022 Cost per call trend: ${analysis.costTrends.costPerCallTrend}`);
}
if (options.performance || !options.usage && !options.costs) {
console.log("\n\u26A1 Performance Trends:");
console.log(`\u2022 Response time trend: ${analysis.performanceTrends.responseTimeTrend}`);
console.log(`\u2022 Error rate trend: ${analysis.performanceTrends.errorRateTrend}`);
console.log(`\u2022 Success rate trend: ${analysis.performanceTrends.successRateTrend}`);
}
}
});
const optimizeCommand = new import_commander.Command("optimize").description("Get actionable recommendations to optimize costs and performance").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "7d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--costs", "Show cost optimization recommendations").option("--performance", "Show performance optimization recommendations").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const recommendations = analytics.getOptimizationRecommendations({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(recommendations, null, 2));
} else if (options.format === "summary") {
const allRecs = [
...recommendations.costOptimization,
...recommendations.performanceOptimization,
...recommendations.usageOptimization
];
allRecs.forEach((rec) => console.log(`\u2022 ${rec}`));
} else {
console.log(`
\u{1F4A1} Optimization Recommendations (Last ${options.since})
`);
console.log("\u2500".repeat(60));
if (options.costs || !options.performance && recommendations.costOptimization.length > 0) {
console.log("\n\u{1F4B0} Cost Optimization:");
recommendations.costOptimization.forEach((rec) => {
console.log(`\u2022 ${rec}`);
});
}
if (options.performance || !options.costs && recommendations.performanceOptimization.length > 0) {
console.log("\n\u26A1 Performance Optimization:");
recommendations.performanceOptimization.forEach((rec) => {
console.log(`\u2022 ${rec}`);
});
}
if (!options.costs && !options.performance && recommendations.usageOptimization.length > 0) {
console.log("\n\u{1F3AF} Usage Optimization:");
recommendations.usageOptimization.forEach((rec) => {
console.log(`\u2022 ${rec}`);
});
}
if (recommendations.costOptimization.length === 0 && recommendations.performanceOptimization.length === 0 && recommendations.usageOptimization.length === 0) {
console.log("\n\u2705 No specific optimization recommendations at this time.");
console.log("Your OpenAI API usage appears to be well-optimized!");
}
}
});
const responsesCommand = new import_commander.Command("responses").description("Analyze OpenAI API response content and quality").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "7d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--errors", "Show detailed error analysis").option("--by-model", "Show response quality by model").option("--size-distribution", "Show response size distribution").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const analysis = analytics.analyzeResponses({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(analysis, null, 2));
} else if (options.format === "csv") {
console.log("Period,Total Responses,Avg Size,Success Rate,Error Rate");
console.log(`${options.since},${analysis.totalResponses},${analysis.averageResponseSize.toFixed(1)},${analysis.errorAnalysis.successRate.toFixed(1)}%,${analysis.errorAnalysis.totalErrors}`);
} else if (options.format === "summary") {
console.log(`\u{1F4C4} Total Responses: ${analysis.totalResponses}`);
console.log(`\u{1F4CF} Avg Response Size: ${analysis.averageResponseSize.toFixed(1)} bytes`);
console.log(`\u2705 Success Rate: ${analysis.errorAnalysis.successRate.toFixed(1)}%`);
console.log(`\u274C Total Errors: ${analysis.errorAnalysis.totalErrors}`);
} else {
console.log(`
\u{1F4C4} Response Analysis (Last ${options.since})
`);
console.log("\u2500".repeat(60));
console.log(`Total Responses: ${analysis.totalResponses}`);
console.log(`Average Response Size: ${analysis.averageResponseSize.toFixed(1)} bytes`);
console.log(`Success Rate: ${analysis.errorAnalysis.successRate.toFixed(1)}%`);
console.log(`Error Rate: ${analysis.errorAnalysis.totalErrors > 0 ? (analysis.errorAnalysis.totalErrors / analysis.totalResponses * 100).toFixed(1) : 0}%`);
console.log("\u2500".repeat(60));
if (options.sizeDistribution) {
console.log("\n\u{1F4CA} Response Size Distribution:");
console.log(`\u2022 Small (< 500 bytes): ${analysis.responseSizeDistribution.small} responses`);
console.log(`\u2022 Medium (500-2000 bytes): ${analysis.responseSizeDistribution.medium} responses`);
console.log(`\u2022 Large (> 2000 bytes): ${analysis.responseSizeDistribution.large} responses`);
}
if (options.errors && analysis.errorAnalysis.totalErrors > 0) {
console.log("\n\u{1F6A8} Error Analysis:");
Object.entries(analysis.errorAnalysis.errorTypes).forEach(([errorType, count]) => {
console.log(`\u2022 ${errorType}: ${count} occurrences`);
});
if (analysis.errorAnalysis.errorMessages.length > 0) {
console.log("\n\u{1F4DD} Recent Error Messages:");
analysis.errorAnalysis.errorMessages.slice(0, 5).forEach((msg) => {
console.log(`\u2022 ${msg}`);
});
}
}
if (options.byModel && Object.keys(analysis.modelResponseQuality).length > 0) {
console.log("\n\u{1F916} Response Quality by Model:");
Object.entries(analysis.modelResponseQuality).forEach(([model, data]) => {
console.log(`${model}:`);
console.log(` \u2022 Avg response length: ${data.averageResponseLength.toFixed(1)} chars`);
console.log(` \u2022 Avg response size: ${data.averageResponseSize.toFixed(1)} bytes`);
console.log(` \u2022 Success rate: ${data.successRate.toFixed(1)}%`);
console.log("");
});
}
}
});
const contentCommand = new import_commander.Command("content").description("Get insights about response content patterns and quality").option("--since <period>", "Time period (1h, 24h, 7d, 30d, 90d, 1y)", "7d").option("--format <format>", "Output format (table, json, csv, summary)", "table").option("--patterns", "Show response length patterns").option("--finish-reasons", "Show finish reason distribution").option("--quality", "Show response quality metrics").action(async (options) => {
const config = getConfig();
const analytics = new Analytics(config.localStorePath);
const insights = analytics.getContentInsights({
since: options.since,
format: options.format
});
if (options.format === "json") {
console.log(JSON.stringify(insights, null, 2));
} else if (options.format === "summary") {
console.log(`\u{1F4CA} Response Patterns: ${insights.responsePatterns.shortResponses + insights.responsePatterns.mediumResponses + insights.responsePatterns.longResponses} total`);
console.log(`\u{1F3AF} Avg tokens/char: ${insights.responseQuality.averageTokensPerCharacter.toFixed(2)}`);
console.log(`\u26A1 Response efficiency: ${insights.responseQuality.responseEfficiency.toFixed(4)}`);
} else {
console.log(`
\u{1F4DD} Content Insights (Last ${options.since})
`);
console.log("\u2500".repeat(60));
if (options.patterns || !options.finishReasons && !options.quality) {
console.log("\n\u{1F4CA} Response Length Patterns:");
const total = insights.responsePatterns.shortResponses + insights.responsePatterns.mediumResponses + insights.responsePatterns.longResponses;
console.log(`\u2022 Short responses (< 50 chars): ${insights.responsePatterns.shortResponses} (${total > 0 ? (insights.responsePatterns.shortResponses / total * 100).toFixed(1) : 0}%)`);
console.log(`\u2022 Medium responses (50-200 chars): ${insights.responsePatterns.mediumResponses} (${total > 0 ? (insights.responsePatterns.mediumResponses / total * 100).toFixed(1) : 0}%)`);
console.log(`\u2022 Long responses (> 200 chars): ${insights.responsePatterns.longResponses} (${total > 0 ? (insights.responsePatterns.longResponses / total * 100).toFixed(1) : 0}%)`);
}
if (options.finishReasons || !options.patterns && !options.quality) {
console.log("\n\u{1F3AF} Finish Reasons:");
Object.entries(insights.finishReasons).forEach(([reason, count]) => {
console.log(`\u2022 ${reason}: ${count} responses`);
});
}
if (options.quality || !options.patterns && !options.finishReasons) {
console.log("\n\u26A1 Response Quality Metrics:");
console.log(`\u2022 Average tokens per character: ${insights.responseQuality.averageTokensPerCharacter.toFixed(2)}`);
console.log(`\u2022 Response efficiency (tokens/byte): ${insights.responseQuality.responseEfficiency.toFixed(4)}`);
}
if (Object.values(insights.commonErrors).some((count) => count > 0)) {
console.log("\n\u{1F6A8} Common Error Types:");
Object.entries(insights.commonErrors).forEach(([errorType, count]) => {
if (count > 0) {
console.log(`\u2022 ${errorType}: ${count} occurrences`);
}
});
}
}
});
commands.push(
costCommand,
usageCommand,
performanceCommand,
modelsCommand,
aggregationsCommand,
trendsCommand,
optimizeCommand,
responsesCommand,
contentCommand
);
return commands;
}
// src/cli/index.ts
var program = new import_commander2.Command();
program.name("spinal").description("Spinal CLI").version("0.1.0");
var store = new import_conf.default({ projectName: "spinal" });
program.command("status").description("Show current mode and configuration summary").action(async () => {
const cfg = getConfig();
const mode = cfg.mode;
const endpoint = cfg.endpoint;
const excluded = process.env.SPINAL_EXCLUDED_HOSTS ?? "api.anthropic.com,api.azure.com";
const excludeOpenAI = process.env.SPINAL_EXCLUDE_OPENAI === "true";
console.log(JSON.stringify({ mode, endpoint, localStorePath: cfg.localStorePath, excludedHosts: excluded, excludeOpenAI }, null, 2));
});
program.command("login").description