langsmith
Version:
Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.
816 lines (815 loc) • 33.3 kB
JavaScript
import { Client, RunTree } from "./index.js";
import { v5 as uuid5 } from "uuid";
import { getCurrentRunTree } from "./singletons/traceable.js";
import { getLangSmithEnvironmentVariable, getEnvironmentVariable, } from "./utils/env.js";
import { isTracingEnabled } from "./env.js";
// Attempt to convert CoreMessage to a LangChain-compatible format
// which allows us to render messages more nicely in LangSmith
function convertCoreToSmith(message) {
if (message.role === "assistant") {
const data = { content: message.content };
if (Array.isArray(message.content)) {
data.content = message.content.map((part) => {
if (part.type === "text") {
return {
type: "text",
text: part.text,
...part.experimental_providerMetadata,
};
}
if (part.type === "tool-call") {
return {
type: "tool_use",
name: part.toolName,
id: part.toolCallId,
input: part.args,
...part.experimental_providerMetadata,
};
}
return part;
});
const toolCalls = message.content.filter((part) => part.type === "tool-call");
if (toolCalls.length > 0) {
data.additional_kwargs ??= {};
data.additional_kwargs.tool_calls = toolCalls.map((part) => {
return {
id: part.toolCallId,
type: "function",
function: {
name: part.toolName,
id: part.toolCallId,
arguments: JSON.stringify(part.args),
},
};
});
}
}
return { type: "ai", data };
}
if (message.role === "user") {
const data = { content: message.content };
if (Array.isArray(message.content)) {
data.content = message.content.map((part) => {
if (part.type === "text") {
return {
type: "text",
text: part.text,
...part.experimental_providerMetadata,
};
}
if (part.type === "image") {
let imageUrl = part.image;
if (typeof imageUrl !== "string") {
let uint8Array;
if (imageUrl != null &&
typeof imageUrl === "object" &&
"type" in imageUrl &&
"data" in imageUrl) {
// Typing is wrong here if a buffer is passed in
uint8Array = new Uint8Array(imageUrl.data);
}
else if (imageUrl != null &&
typeof imageUrl === "object" &&
Object.keys(imageUrl).every((key) => !isNaN(Number(key)))) {
// ArrayBuffers get turned into objects with numeric keys for some reason
uint8Array = new Uint8Array(Array.from({
...imageUrl,
length: Object.keys(imageUrl).length,
}));
}
if (uint8Array) {
let binary = "";
for (let i = 0; i < uint8Array.length; i++) {
binary += String.fromCharCode(uint8Array[i]);
}
imageUrl = btoa(binary);
}
}
return {
type: "image_url",
image_url: imageUrl,
...part.experimental_providerMetadata,
};
}
return part;
});
}
return { type: "human", data };
}
if (message.role === "system") {
return { type: "system", data: { content: message.content } };
}
if (message.role === "tool") {
const res = message.content.map((toolCall) => {
return {
type: "tool",
data: {
content: JSON.stringify(toolCall.result),
name: toolCall.toolName,
tool_call_id: toolCall.toolCallId,
},
};
});
if (res.length === 1)
return res[0];
return res;
}
return message;
}
const tryJson = (str) => {
try {
if (!str)
return str;
if (typeof str !== "string")
return str;
return JSON.parse(str);
}
catch {
return str;
}
};
function stripNonAlphanumeric(input) {
return input.replace(/[-:.]/g, "");
}
function getDotOrder(item) {
const { startTime: [seconds, nanoseconds], id: runId, executionOrder, } = item;
// Date only has millisecond precision, so we use the microseconds to break
// possible ties, avoiding incorrect run order
const nanosecondString = String(nanoseconds).padStart(9, "0");
const msFull = Number(nanosecondString.slice(0, 6)) + executionOrder;
const msString = String(msFull).padStart(6, "0");
const ms = Number(msString.slice(0, -3));
const ns = msString.slice(-3);
return (stripNonAlphanumeric(`${new Date(seconds * 1000 + ms).toISOString().slice(0, -1)}${ns}Z`) + runId);
}
function joinDotOrder(...segments) {
return segments.filter(Boolean).join(".");
}
function removeDotOrder(dotOrder, ...ids) {
return dotOrder
.split(".")
.filter((i) => !ids.some((id) => i.includes(id)))
.join(".");
}
function reparentDotOrder(dotOrder, sourceRunId, parentDotOrder) {
const segments = dotOrder.split(".");
const sourceIndex = segments.findIndex((i) => i.includes(sourceRunId));
if (sourceIndex === -1)
return dotOrder;
return joinDotOrder(...parentDotOrder.split("."), ...segments.slice(sourceIndex));
}
function getMutableRunCreate(dotOrder) {
const segments = dotOrder.split(".").map((i) => {
const [startTime, runId] = i.split("Z");
return { startTime, runId };
});
const traceId = segments[0].runId;
const parentRunId = segments.at(-2)?.runId;
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const runId = segments.at(-1).runId;
return {
id: runId,
trace_id: traceId,
dotted_order: dotOrder,
parent_run_id: parentRunId,
};
}
function convertToTimestamp([seconds, nanoseconds]) {
const ms = String(nanoseconds).slice(0, 3);
return Number(String(seconds) + ms);
}
function sortByHr(a, b) {
if (a.startTime[0] !== b.startTime[0]) {
return Math.sign(a.startTime[0] - b.startTime[0]);
}
else if (a.startTime[1] !== b.startTime[1]) {
return Math.sign(a.startTime[1] - b.startTime[1]);
}
else if (getParentSpanId(a) === b.spanContext().spanId) {
return -1;
}
else if (getParentSpanId(b) === a.spanContext().spanId) {
return 1;
}
else {
return 0;
}
}
const ROOT = "$";
const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47";
const RUN_ID_METADATA_KEY = {
input: "langsmith:runId",
output: "ai.telemetry.metadata.langsmith:runId",
};
const RUN_NAME_METADATA_KEY = {
input: "langsmith:runName",
output: "ai.telemetry.metadata.langsmith:runName",
};
const TRACE_METADATA_KEY = {
input: "langsmith:trace",
output: "ai.telemetry.metadata.langsmith:trace",
};
const BAGGAGE_METADATA_KEY = {
input: "langsmith:baggage",
output: "ai.telemetry.metadata.langsmith:baggage",
};
const RESERVED_METADATA_KEYS = [
RUN_ID_METADATA_KEY.output,
RUN_NAME_METADATA_KEY.output,
TRACE_METADATA_KEY.output,
BAGGAGE_METADATA_KEY.output,
];
function getParentSpanId(span) {
// Backcompat shim to support OTEL 1.x and 2.x
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return (span.parentSpanId ?? span.parentSpanContext?.spanId ?? undefined);
}
/**
* OpenTelemetry trace exporter for Vercel AI SDK.
*
* @example
* ```ts
* import { AISDKExporter } from "langsmith/vercel";
* import { Client } from "langsmith";
*
* import { generateText } from "ai";
* import { openai } from "@ai-sdk/openai";
*
* import { NodeSDK } from "@opentelemetry/sdk-node";
* import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
*
* const client = new Client();
*
* const sdk = new NodeSDK({
* traceExporter: new AISDKExporter({ client }),
* instrumentations: [getNodeAutoInstrumentations()],
* });
*
* sdk.start();
*
* const res = await generateText({
* model: openai("gpt-4o-mini"),
* messages: [
* {
* role: "user",
* content: "What color is the sky?",
* },
* ],
* experimental_telemetry: AISDKExporter.getSettings({
* runName: "langsmith_traced_call",
* metadata: { userId: "123", language: "english" },
* }),
* });
*
* await sdk.shutdown();
* ```
*/
export class AISDKExporter {
constructor(args) {
Object.defineProperty(this, "client", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "traceByMap", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
Object.defineProperty(this, "seenSpanInfo", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
Object.defineProperty(this, "pendingSpans", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
Object.defineProperty(this, "debug", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "projectName", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
/** @internal */
Object.defineProperty(this, "getSpanAttributeKey", {
enumerable: true,
configurable: true,
writable: true,
value: (span, key) => {
const attributes = span.attributes;
return key in attributes && typeof attributes[key] === "string"
? attributes[key]
: undefined;
}
});
this.client = args?.client ?? new Client();
this.debug =
args?.debug ?? getEnvironmentVariable("OTEL_LOG_LEVEL") === "DEBUG";
this.projectName = args?.projectName;
this.logDebug("creating exporter", { tracingEnabled: isTracingEnabled() });
}
static getSettings(settings) {
const { runId, runName, ...rest } = settings ?? {};
const metadata = { ...rest?.metadata };
if (runId != null)
metadata[RUN_ID_METADATA_KEY.input] = runId;
if (runName != null)
metadata[RUN_NAME_METADATA_KEY.input] = runName;
// attempt to obtain the run tree if used within a traceable function
let defaultEnabled = settings?.isEnabled ?? isTracingEnabled();
try {
const runTree = getCurrentRunTree();
const headers = runTree.toHeaders();
metadata[TRACE_METADATA_KEY.input] = headers["langsmith-trace"];
metadata[BAGGAGE_METADATA_KEY.input] = headers["baggage"];
// honor the tracingEnabled flag if coming from traceable
if (runTree.tracingEnabled != null) {
defaultEnabled = runTree.tracingEnabled;
}
}
catch {
// pass
}
if (metadata[RUN_ID_METADATA_KEY.input] &&
metadata[TRACE_METADATA_KEY.input]) {
throw new Error("Cannot provide `runId` when used within traceable function.");
}
return { ...rest, isEnabled: rest.isEnabled ?? defaultEnabled, metadata };
}
/** @internal */
parseInteropFromMetadata(span, parentSpan) {
if (!this.isRootRun(span))
return undefined;
if (parentSpan?.name === "ai.toolCall") {
return undefined;
}
const userTraceId = this.getSpanAttributeKey(span, RUN_ID_METADATA_KEY.output);
const parentTrace = this.getSpanAttributeKey(span, TRACE_METADATA_KEY.output);
if (parentTrace && userTraceId) {
throw new Error(`Cannot provide both "${RUN_ID_METADATA_KEY.input}" and "${TRACE_METADATA_KEY.input}" metadata keys.`);
}
if (parentTrace) {
const parentRunTree = RunTree.fromHeaders({
"langsmith-trace": parentTrace,
baggage: this.getSpanAttributeKey(span, BAGGAGE_METADATA_KEY.output) || "",
});
if (!parentRunTree)
throw new Error("Unreachable code: empty parent run tree");
return { type: "traceable", parentRunTree };
}
if (userTraceId)
return { type: "user", userRunId: userTraceId };
return undefined;
}
/** @internal */
getRunCreate(span, projectName) {
const asRunCreate = (rawConfig) => {
const aiMetadata = Object.keys(span.attributes)
.filter((key) => key.startsWith("ai.telemetry.metadata.") &&
!RESERVED_METADATA_KEYS.includes(key))
.reduce((acc, key) => {
acc[key.slice("ai.telemetry.metadata.".length)] =
span.attributes[key];
return acc;
}, {});
if (("ai.telemetry.functionId" in span.attributes &&
span.attributes["ai.telemetry.functionId"]) ||
("resource.name" in span.attributes && span.attributes["resource.name"])) {
aiMetadata["functionId"] =
span.attributes["ai.telemetry.functionId"] ||
span.attributes["resource.name"];
}
const parsedStart = convertToTimestamp(span.startTime);
const parsedEnd = convertToTimestamp(span.endTime);
let name = rawConfig.name;
// if user provided a custom name, only use it if it's the root
if (this.isRootRun(span)) {
name =
this.getSpanAttributeKey(span, RUN_NAME_METADATA_KEY.output) || name;
}
const config = {
...rawConfig,
name,
extra: {
...rawConfig.extra,
metadata: {
...rawConfig.extra?.metadata,
...aiMetadata,
"ai.operationId": span.attributes["ai.operationId"],
},
},
session_name: projectName ??
this.projectName ??
getLangSmithEnvironmentVariable("PROJECT") ??
getLangSmithEnvironmentVariable("SESSION"),
start_time: Math.min(parsedStart, parsedEnd),
end_time: Math.max(parsedStart, parsedEnd),
};
return config;
};
switch (span.name) {
case "ai.generateText.doGenerate":
case "ai.generateText":
case "ai.streamText.doStream":
case "ai.streamText": {
const inputs = (() => {
if ("ai.prompt.messages" in span.attributes) {
return {
messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap((i) => convertCoreToSmith(i)),
};
}
if ("ai.prompt" in span.attributes) {
const input = tryJson(span.attributes["ai.prompt"]);
if (typeof input === "object" &&
input != null &&
"messages" in input &&
Array.isArray(input.messages)) {
return {
messages: input.messages.flatMap((i) => convertCoreToSmith(i)),
};
}
return { input };
}
return {};
})();
const outputs = (() => {
let result = undefined;
if (span.attributes["ai.response.toolCalls"]) {
let content = tryJson(span.attributes["ai.response.toolCalls"]);
if (Array.isArray(content)) {
content = content.map((i) => ({
type: "tool-call",
...i,
args: tryJson(i.args),
}));
}
result = {
llm_output: convertCoreToSmith({
role: "assistant",
content,
}),
};
}
else if (span.attributes["ai.response.text"]) {
result = {
llm_output: convertCoreToSmith({
role: "assistant",
content: span.attributes["ai.response.text"],
}),
};
}
if (span.attributes["ai.usage.completionTokens"]) {
result ??= {};
result.llm_output ??= {};
result.llm_output.token_usage ??= {};
result.llm_output.token_usage["completion_tokens"] =
span.attributes["ai.usage.completionTokens"];
}
if (span.attributes["ai.usage.promptTokens"]) {
result ??= {};
result.llm_output ??= {};
result.llm_output.token_usage ??= {};
result.llm_output.token_usage["prompt_tokens"] =
span.attributes["ai.usage.promptTokens"];
}
return result;
})();
const invocationParams = (() => {
if ("ai.prompt.tools" in span.attributes) {
return {
tools: span.attributes["ai.prompt.tools"].flatMap((tool) => {
try {
return JSON.parse(tool);
}
catch {
// pass
}
return [];
}),
};
}
return {};
})();
const events = [];
const firstChunkEvent = span.events.find((i) => i.name === "ai.stream.firstChunk");
if (firstChunkEvent) {
events.push({
name: "new_token",
time: convertToTimestamp(firstChunkEvent.time),
});
}
// TODO: add first_token_time
return asRunCreate({
run_type: "llm",
name: span.attributes["ai.model.provider"],
inputs,
outputs,
events,
extra: {
invocation_params: invocationParams,
batch_size: 1,
metadata: {
ls_provider: span.attributes["ai.model.provider"]
.split(".")
.at(0),
ls_model_type: span.attributes["ai.model.provider"]
.split(".")
.at(1),
ls_model_name: span.attributes["ai.model.id"],
},
},
});
}
case "ai.toolCall": {
const args = tryJson(span.attributes["ai.toolCall.args"]);
let inputs = { args };
if (typeof args === "object" && args != null) {
inputs = args;
}
const output = tryJson(span.attributes["ai.toolCall.result"]);
let outputs = { output };
if (typeof output === "object" && output != null) {
outputs = output;
}
return asRunCreate({
run_type: "tool",
name: span.attributes["ai.toolCall.name"],
inputs,
outputs,
});
}
case "ai.streamObject":
case "ai.streamObject.doStream":
case "ai.generateObject":
case "ai.generateObject.doGenerate": {
const inputs = (() => {
if ("ai.prompt.messages" in span.attributes) {
return {
messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap((i) => convertCoreToSmith(i)),
};
}
if ("ai.prompt" in span.attributes) {
return { input: tryJson(span.attributes["ai.prompt"]) };
}
return {};
})();
const outputs = (() => {
let result = undefined;
if (span.attributes["ai.response.object"]) {
result = {
output: tryJson(span.attributes["ai.response.object"]),
};
}
if (span.attributes["ai.usage.completionTokens"]) {
result ??= {};
result.llm_output ??= {};
result.llm_output.token_usage ??= {};
result.llm_output.token_usage["completion_tokens"] =
span.attributes["ai.usage.completionTokens"];
}
if (span.attributes["ai.usage.promptTokens"]) {
result ??= {};
result.llm_output ??= {};
result.llm_output.token_usage ??= {};
result.llm_output.token_usage["prompt_tokens"] =
+span.attributes["ai.usage.promptTokens"];
}
return result;
})();
const events = [];
const firstChunkEvent = span.events.find((i) => i.name === "ai.stream.firstChunk");
if (firstChunkEvent) {
events.push({
name: "new_token",
time: convertToTimestamp(firstChunkEvent.time),
});
}
return asRunCreate({
run_type: "llm",
name: span.attributes["ai.model.provider"],
inputs,
outputs,
events,
extra: {
batch_size: 1,
metadata: {
ls_provider: span.attributes["ai.model.provider"]
.split(".")
.at(0),
ls_model_type: span.attributes["ai.model.provider"]
.split(".")
.at(1),
ls_model_name: span.attributes["ai.model.id"],
},
},
});
}
case "ai.embed":
case "ai.embed.doEmbed":
case "ai.embedMany":
case "ai.embedMany.doEmbed":
default:
return undefined;
}
}
/** @internal */
isRootRun(span) {
switch (span.name) {
case "ai.generateText":
case "ai.streamText":
case "ai.generateObject":
case "ai.streamObject":
case "ai.embed":
case "ai.embedMany":
return true;
default:
return false;
}
}
_export(spans, resultCallback) {
this.logDebug("exporting spans", spans);
const typedSpans = spans
.concat(Object.values(this.pendingSpans))
.slice()
// Parent spans should go before child spans in the final order,
// but may have the same exact start time as their children.
// They will end earlier, so break ties by end time.
// TODO: Figure out why this happens.
.sort((a, b) => sortByHr(a, b));
for (const span of typedSpans) {
const { traceId, spanId } = span.spanContext();
const runId = uuid5(spanId, RUN_ID_NAMESPACE);
let parentId = getParentSpanId(span);
let parentRunId = parentId
? uuid5(parentId, RUN_ID_NAMESPACE)
: undefined;
let parentSpanInfo = parentRunId
? this.seenSpanInfo[parentRunId]
: undefined;
// Unrelated, untraced spans should behave as passthroughs from LangSmith's perspective.
while (parentSpanInfo != null &&
this.getRunCreate(parentSpanInfo.span) == null) {
parentId = getParentSpanId(parentSpanInfo.span);
if (parentId == null) {
break;
}
parentRunId = parentId ? uuid5(parentId, RUN_ID_NAMESPACE) : undefined;
parentSpanInfo = parentRunId
? this.seenSpanInfo[parentRunId]
: undefined;
}
// Export may be called in any order, so we need to queue any spans with missing parents
// for retry later in order to determine whether their parents are tool calls
// and should not be reparented below.
if (parentRunId !== undefined && parentSpanInfo === undefined) {
this.pendingSpans[spanId] = span;
continue;
}
else {
delete this.pendingSpans[spanId];
}
this.traceByMap[traceId] ??= {
childMap: {},
nodeMap: {},
relativeExecutionOrder: {},
};
const traceMap = this.traceByMap[traceId];
traceMap.relativeExecutionOrder[parentRunId ?? ROOT] ??= -1;
traceMap.relativeExecutionOrder[parentRunId ?? ROOT] += 1;
const interop = this.parseInteropFromMetadata(span, parentSpanInfo?.span);
const projectName = (interop?.type === "traceable"
? interop.parentRunTree.project_name
: undefined) ?? parentSpanInfo?.projectName;
const run = this.getRunCreate(span, projectName);
traceMap.nodeMap[runId] ??= {
id: runId,
startTime: span.startTime,
run,
sent: false,
interop,
executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? ROOT],
};
if (this.seenSpanInfo[runId] == null) {
this.seenSpanInfo[runId] = {
span,
dotOrder: joinDotOrder(parentSpanInfo?.dotOrder, getDotOrder(traceMap.nodeMap[runId])),
projectName,
sent: false,
};
}
if (this.debug)
console.log(`[${span.name}] ${runId}`, run);
traceMap.childMap[parentRunId ?? ROOT] ??= [];
traceMap.childMap[parentRunId ?? ROOT].push(traceMap.nodeMap[runId]);
}
const sampled = [];
const actions = [];
for (const traceId of Object.keys(this.traceByMap)) {
const traceMap = this.traceByMap[traceId];
const queue = Object.keys(traceMap.childMap)
.map((runId) => {
if (runId === ROOT) {
return traceMap.childMap[runId];
}
return [];
})
.flat();
const seen = new Set();
while (queue.length) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const task = queue.shift();
if (seen.has(task.id))
continue;
let taskDotOrder = this.seenSpanInfo[task.id].dotOrder;
if (!task.sent) {
if (task.run != null) {
if (task.interop?.type === "user") {
actions.push({
type: "rename",
sourceRunId: task.id,
targetRunId: task.interop.userRunId,
});
}
if (task.interop?.type === "traceable") {
actions.push({
type: "reparent",
runId: task.id,
parentDotOrder: task.interop.parentRunTree.dotted_order,
});
}
for (const action of actions) {
if (action.type === "delete") {
taskDotOrder = removeDotOrder(taskDotOrder, action.runId);
}
if (action.type === "reparent") {
taskDotOrder = reparentDotOrder(taskDotOrder, action.runId, action.parentDotOrder);
}
if (action.type === "rename") {
taskDotOrder = taskDotOrder.replace(action.sourceRunId, action.targetRunId);
}
}
this.seenSpanInfo[task.id].dotOrder = taskDotOrder;
if (!this.seenSpanInfo[task.id].sent) {
sampled.push({
...task.run,
...getMutableRunCreate(taskDotOrder),
});
}
this.seenSpanInfo[task.id].sent = true;
}
else {
actions.push({ type: "delete", runId: task.id });
}
task.sent = true;
}
const children = traceMap.childMap[task.id] ?? [];
queue.push(...children);
}
}
this.logDebug(`sampled runs to be sent to LangSmith`, sampled);
Promise.all(sampled.map((run) => this.client.createRun(run))).then(() => resultCallback({ code: 0 }), (error) => resultCallback({ code: 1, error }));
}
export(spans, resultCallback) {
this._export(spans, (result) => {
if (result.code === 0) {
// Empty export to try flushing pending spans to rule out any trace order shenanigans
this._export([], resultCallback);
}
else {
resultCallback(result);
}
});
}
async shutdown() {
// find nodes which are incomplete
const incompleteNodes = Object.values(this.traceByMap).flatMap((trace) => Object.values(trace.nodeMap).filter((i) => !i.sent && i.run != null));
this.logDebug("shutting down", { incompleteNodes });
if (incompleteNodes.length > 0) {
console.warn("Some incomplete nodes were found before shutdown and not sent to LangSmith.");
}
await this.forceFlush();
}
async forceFlush() {
await new Promise((resolve) => {
this.export([], resolve);
});
await this.client.awaitPendingTraceBatches();
}
logDebug(...args) {
if (!this.debug)
return;
console.debug(`[${new Date().toISOString()}] [LangSmith]`, ...args);
}
}