langsmith
Version:
Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.
1,282 lines • 138 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.Client = exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES = exports.AutoBatchQueue = void 0;
exports.mergeRuntimeEnvIntoRunCreate = mergeRuntimeEnvIntoRunCreate;
const uuid = __importStar(require("uuid"));
const async_caller_js_1 = require("./utils/async_caller.cjs");
const messages_js_1 = require("./utils/messages.cjs");
const env_js_1 = require("./utils/env.cjs");
const index_js_1 = require("./index.cjs");
const _uuid_js_1 = require("./utils/_uuid.cjs");
const warn_js_1 = require("./utils/warn.cjs");
const prompts_js_1 = require("./utils/prompts.cjs");
const error_js_1 = require("./utils/error.cjs");
const fetch_js_1 = require("./singletons/fetch.cjs");
const index_js_2 = require("./utils/fast-safe-stringify/index.cjs");
function mergeRuntimeEnvIntoRunCreate(run) {
const runtimeEnv = (0, env_js_1.getRuntimeEnvironment)();
const envVars = (0, env_js_1.getLangChainEnvVarsMetadata)();
const extra = run.extra ?? {};
const metadata = extra.metadata;
run.extra = {
...extra,
runtime: {
...runtimeEnv,
...extra?.runtime,
},
metadata: {
...envVars,
...(envVars.revision_id || run.revision_id
? { revision_id: run.revision_id ?? envVars.revision_id }
: {}),
...metadata,
},
};
return run;
}
const getTracingSamplingRate = (configRate) => {
const samplingRateStr = configRate?.toString() ??
(0, env_js_1.getLangSmithEnvironmentVariable)("TRACING_SAMPLING_RATE");
if (samplingRateStr === undefined) {
return undefined;
}
const samplingRate = parseFloat(samplingRateStr);
if (samplingRate < 0 || samplingRate > 1) {
throw new Error(`LANGSMITH_TRACING_SAMPLING_RATE must be between 0 and 1 if set. Got: ${samplingRate}`);
}
return samplingRate;
};
// utility functions
const isLocalhost = (url) => {
const strippedUrl = url.replace("http://", "").replace("https://", "");
const hostname = strippedUrl.split("/")[0].split(":")[0];
return (hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1");
};
async function toArray(iterable) {
const result = [];
for await (const item of iterable) {
result.push(item);
}
return result;
}
function trimQuotes(str) {
if (str === undefined) {
return undefined;
}
return str
.trim()
.replace(/^"(.*)"$/, "$1")
.replace(/^'(.*)'$/, "$1");
}
const handle429 = async (response) => {
if (response?.status === 429) {
const retryAfter = parseInt(response.headers.get("retry-after") ?? "30", 10) * 1000;
if (retryAfter > 0) {
await new Promise((resolve) => setTimeout(resolve, retryAfter));
// Return directly after calling this check
return true;
}
}
// Fall back to existing status checks
return false;
};
function _formatFeedbackScore(score) {
if (typeof score === "number") {
// Truncate at 4 decimal places
return Number(score.toFixed(4));
}
return score;
}
class AutoBatchQueue {
constructor() {
Object.defineProperty(this, "items", {
enumerable: true,
configurable: true,
writable: true,
value: []
});
Object.defineProperty(this, "sizeBytes", {
enumerable: true,
configurable: true,
writable: true,
value: 0
});
}
peek() {
return this.items[0];
}
push(item) {
let itemPromiseResolve;
const itemPromise = new Promise((resolve) => {
// Setting itemPromiseResolve is synchronous with promise creation:
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/Promise
itemPromiseResolve = resolve;
});
const size = (0, index_js_2.serialize)(item.item).length;
this.items.push({
action: item.action,
payload: item.item,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
itemPromiseResolve: itemPromiseResolve,
itemPromise,
size,
});
this.sizeBytes += size;
return itemPromise;
}
pop(upToSizeBytes) {
if (upToSizeBytes < 1) {
throw new Error("Number of bytes to pop off may not be less than 1.");
}
const popped = [];
let poppedSizeBytes = 0;
// Pop items until we reach or exceed the size limit
while (poppedSizeBytes + (this.peek()?.size ?? 0) < upToSizeBytes &&
this.items.length > 0) {
const item = this.items.shift();
if (item) {
popped.push(item);
poppedSizeBytes += item.size;
this.sizeBytes -= item.size;
}
}
// If there is an item on the queue we were unable to pop,
// just return it as a single batch.
if (popped.length === 0 && this.items.length > 0) {
const item = this.items.shift();
popped.push(item);
poppedSizeBytes += item.size;
this.sizeBytes -= item.size;
}
return [
popped.map((it) => ({ action: it.action, item: it.payload })),
() => popped.forEach((it) => it.itemPromiseResolve()),
];
}
}
exports.AutoBatchQueue = AutoBatchQueue;
// 20 MB
exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20_971_520;
const SERVER_INFO_REQUEST_TIMEOUT = 2500;
class Client {
constructor(config = {}) {
Object.defineProperty(this, "apiKey", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "apiUrl", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "webUrl", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "caller", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "batchIngestCaller", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "timeout_ms", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "_tenantId", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "hideInputs", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "hideOutputs", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "tracingSampleRate", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "filteredPostUuids", {
enumerable: true,
configurable: true,
writable: true,
value: new Set()
});
Object.defineProperty(this, "autoBatchTracing", {
enumerable: true,
configurable: true,
writable: true,
value: true
});
Object.defineProperty(this, "autoBatchQueue", {
enumerable: true,
configurable: true,
writable: true,
value: new AutoBatchQueue()
});
Object.defineProperty(this, "autoBatchTimeout", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "autoBatchAggregationDelayMs", {
enumerable: true,
configurable: true,
writable: true,
value: 250
});
Object.defineProperty(this, "batchSizeBytesLimit", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "fetchOptions", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "settings", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "blockOnRootRunFinalization", {
enumerable: true,
configurable: true,
writable: true,
value: (0, env_js_1.getEnvironmentVariable)("LANGSMITH_TRACING_BACKGROUND") === "false"
});
Object.defineProperty(this, "traceBatchConcurrency", {
enumerable: true,
configurable: true,
writable: true,
value: 5
});
Object.defineProperty(this, "_serverInfo", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Object.defineProperty(this, "_getServerInfoPromise", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "manualFlushMode", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
const defaultConfig = Client.getDefaultClientConfig();
this.tracingSampleRate = getTracingSamplingRate(config.tracingSamplingRate);
this.apiUrl = trimQuotes(config.apiUrl ?? defaultConfig.apiUrl) ?? "";
if (this.apiUrl.endsWith("/")) {
this.apiUrl = this.apiUrl.slice(0, -1);
}
this.apiKey = trimQuotes(config.apiKey ?? defaultConfig.apiKey);
this.webUrl = trimQuotes(config.webUrl ?? defaultConfig.webUrl);
if (this.webUrl?.endsWith("/")) {
this.webUrl = this.webUrl.slice(0, -1);
}
this.timeout_ms = config.timeout_ms ?? 90_000;
this.caller = new async_caller_js_1.AsyncCaller(config.callerOptions ?? {});
this.traceBatchConcurrency =
config.traceBatchConcurrency ?? this.traceBatchConcurrency;
if (this.traceBatchConcurrency < 1) {
throw new Error("Trace batch concurrency must be positive.");
}
this.batchIngestCaller = new async_caller_js_1.AsyncCaller({
maxRetries: 2,
maxConcurrency: this.traceBatchConcurrency,
...(config.callerOptions ?? {}),
onFailedResponseHook: handle429,
});
this.hideInputs =
config.hideInputs ?? config.anonymizer ?? defaultConfig.hideInputs;
this.hideOutputs =
config.hideOutputs ?? config.anonymizer ?? defaultConfig.hideOutputs;
this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
this.blockOnRootRunFinalization =
config.blockOnRootRunFinalization ?? this.blockOnRootRunFinalization;
this.batchSizeBytesLimit = config.batchSizeBytesLimit;
this.fetchOptions = config.fetchOptions || {};
this.manualFlushMode = config.manualFlushMode ?? this.manualFlushMode;
}
static getDefaultClientConfig() {
const apiKey = (0, env_js_1.getLangSmithEnvironmentVariable)("API_KEY");
const apiUrl = (0, env_js_1.getLangSmithEnvironmentVariable)("ENDPOINT") ??
"https://api.smith.langchain.com";
const hideInputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_INPUTS") === "true";
const hideOutputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_OUTPUTS") === "true";
return {
apiUrl: apiUrl,
apiKey: apiKey,
webUrl: undefined,
hideInputs: hideInputs,
hideOutputs: hideOutputs,
};
}
getHostUrl() {
if (this.webUrl) {
return this.webUrl;
}
else if (isLocalhost(this.apiUrl)) {
this.webUrl = "http://localhost:3000";
return this.webUrl;
}
else if (this.apiUrl.endsWith("/api/v1")) {
this.webUrl = this.apiUrl.replace("/api/v1", "");
return this.webUrl;
}
else if (this.apiUrl.includes("/api") &&
!this.apiUrl.split(".", 1)[0].endsWith("api")) {
this.webUrl = this.apiUrl.replace("/api", "");
return this.webUrl;
}
else if (this.apiUrl.split(".", 1)[0].includes("dev")) {
this.webUrl = "https://dev.smith.langchain.com";
return this.webUrl;
}
else if (this.apiUrl.split(".", 1)[0].includes("eu")) {
this.webUrl = "https://eu.smith.langchain.com";
return this.webUrl;
}
else if (this.apiUrl.split(".", 1)[0].includes("beta")) {
this.webUrl = "https://beta.smith.langchain.com";
return this.webUrl;
}
else {
this.webUrl = "https://smith.langchain.com";
return this.webUrl;
}
}
get headers() {
const headers = {
"User-Agent": `langsmith-js/${index_js_1.__version__}`,
};
if (this.apiKey) {
headers["x-api-key"] = `${this.apiKey}`;
}
return headers;
}
processInputs(inputs) {
if (this.hideInputs === false) {
return inputs;
}
if (this.hideInputs === true) {
return {};
}
if (typeof this.hideInputs === "function") {
return this.hideInputs(inputs);
}
return inputs;
}
processOutputs(outputs) {
if (this.hideOutputs === false) {
return outputs;
}
if (this.hideOutputs === true) {
return {};
}
if (typeof this.hideOutputs === "function") {
return this.hideOutputs(outputs);
}
return outputs;
}
prepareRunCreateOrUpdateInputs(run) {
const runParams = { ...run };
if (runParams.inputs !== undefined) {
runParams.inputs = this.processInputs(runParams.inputs);
}
if (runParams.outputs !== undefined) {
runParams.outputs = this.processOutputs(runParams.outputs);
}
return runParams;
}
async _getResponse(path, queryParams) {
const paramsString = queryParams?.toString() ?? "";
const url = `${this.apiUrl}${path}?${paramsString}`;
const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), url, {
method: "GET",
headers: this.headers,
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(response, `Failed to fetch ${path}`);
return response;
}
async _get(path, queryParams) {
const response = await this._getResponse(path, queryParams);
return response.json();
}
async *_getPaginated(path, queryParams = new URLSearchParams(), transform) {
let offset = Number(queryParams.get("offset")) || 0;
const limit = Number(queryParams.get("limit")) || 100;
while (true) {
queryParams.set("offset", String(offset));
queryParams.set("limit", String(limit));
const url = `${this.apiUrl}${path}?${queryParams}`;
const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), url, {
method: "GET",
headers: this.headers,
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(response, `Failed to fetch ${path}`);
const items = transform
? transform(await response.json())
: await response.json();
if (items.length === 0) {
break;
}
yield items;
if (items.length < limit) {
break;
}
offset += items.length;
}
}
async *_getCursorPaginatedList(path, body = null, requestMethod = "POST", dataKey = "runs") {
const bodyParams = body ? { ...body } : {};
while (true) {
const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}${path}`, {
method: requestMethod,
headers: { ...this.headers, "Content-Type": "application/json" },
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
body: JSON.stringify(bodyParams),
});
const responseBody = await response.json();
if (!responseBody) {
break;
}
if (!responseBody[dataKey]) {
break;
}
yield responseBody[dataKey];
const cursors = responseBody.cursors;
if (!cursors) {
break;
}
if (!cursors.next) {
break;
}
bodyParams.cursor = cursors.next;
}
}
// Allows mocking for tests
_shouldSample() {
if (this.tracingSampleRate === undefined) {
return true;
}
return Math.random() < this.tracingSampleRate;
}
_filterForSampling(runs, patch = false) {
if (this.tracingSampleRate === undefined) {
return runs;
}
if (patch) {
const sampled = [];
for (const run of runs) {
if (!this.filteredPostUuids.has(run.id)) {
sampled.push(run);
}
else {
this.filteredPostUuids.delete(run.id);
}
}
return sampled;
}
else {
// For new runs, sample at trace level to maintain consistency
const sampled = [];
for (const run of runs) {
const traceId = run.trace_id ?? run.id;
// If we've already made a decision about this trace, follow it
if (this.filteredPostUuids.has(traceId)) {
continue;
}
// For new traces, apply sampling
if (run.id === traceId) {
if (this._shouldSample()) {
sampled.push(run);
}
else {
this.filteredPostUuids.add(traceId);
}
}
else {
// Child runs follow their trace's sampling decision
sampled.push(run);
}
}
return sampled;
}
}
async _getBatchSizeLimitBytes() {
const serverInfo = await this._ensureServerInfo();
return (this.batchSizeBytesLimit ??
serverInfo.batch_ingest_config?.size_limit_bytes ??
exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES);
}
async _getMultiPartSupport() {
const serverInfo = await this._ensureServerInfo();
return (serverInfo.instance_flags?.dataset_examples_multipart_enabled ?? false);
}
drainAutoBatchQueue(batchSizeLimit) {
const promises = [];
while (this.autoBatchQueue.items.length > 0) {
const [batch, done] = this.autoBatchQueue.pop(batchSizeLimit);
if (!batch.length) {
done();
break;
}
const batchPromise = this._processBatch(batch, done).catch(console.error);
promises.push(batchPromise);
}
return Promise.all(promises);
}
async _processBatch(batch, done) {
if (!batch.length) {
done();
return;
}
try {
const ingestParams = {
runCreates: batch
.filter((item) => item.action === "create")
.map((item) => item.item),
runUpdates: batch
.filter((item) => item.action === "update")
.map((item) => item.item),
};
const serverInfo = await this._ensureServerInfo();
if (serverInfo?.batch_ingest_config?.use_multipart_endpoint) {
await this.multipartIngestRuns(ingestParams);
}
else {
await this.batchIngestRuns(ingestParams);
}
}
finally {
done();
}
}
async processRunOperation(item) {
clearTimeout(this.autoBatchTimeout);
this.autoBatchTimeout = undefined;
if (item.action === "create") {
item.item = mergeRuntimeEnvIntoRunCreate(item.item);
}
const itemPromise = this.autoBatchQueue.push(item);
if (this.manualFlushMode) {
// Rely on manual flushing in serverless environments
return itemPromise;
}
const sizeLimitBytes = await this._getBatchSizeLimitBytes();
if (this.autoBatchQueue.sizeBytes > sizeLimitBytes) {
void this.drainAutoBatchQueue(sizeLimitBytes);
}
if (this.autoBatchQueue.items.length > 0) {
this.autoBatchTimeout = setTimeout(() => {
this.autoBatchTimeout = undefined;
void this.drainAutoBatchQueue(sizeLimitBytes);
}, this.autoBatchAggregationDelayMs);
}
return itemPromise;
}
async _getServerInfo() {
const response = await (0, fetch_js_1._getFetchImplementation)()(`${this.apiUrl}/info`, {
method: "GET",
headers: { Accept: "application/json" },
signal: AbortSignal.timeout(SERVER_INFO_REQUEST_TIMEOUT),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(response, "get server info");
return response.json();
}
async _ensureServerInfo() {
if (this._getServerInfoPromise === undefined) {
this._getServerInfoPromise = (async () => {
if (this._serverInfo === undefined) {
try {
this._serverInfo = await this._getServerInfo();
}
catch (e) {
console.warn(`[WARNING]: LangSmith failed to fetch info on supported operations. Falling back to batch operations and default limits.`);
}
}
return this._serverInfo ?? {};
})();
}
return this._getServerInfoPromise.then((serverInfo) => {
if (this._serverInfo === undefined) {
this._getServerInfoPromise = undefined;
}
return serverInfo;
});
}
async _getSettings() {
if (!this.settings) {
this.settings = this._get("/settings");
}
return await this.settings;
}
/**
* Flushes current queued traces.
*/
async flush() {
const sizeLimitBytes = await this._getBatchSizeLimitBytes();
await this.drainAutoBatchQueue(sizeLimitBytes);
}
async createRun(run) {
if (!this._filterForSampling([run]).length) {
return;
}
const headers = { ...this.headers, "Content-Type": "application/json" };
const session_name = run.project_name;
delete run.project_name;
const runCreate = this.prepareRunCreateOrUpdateInputs({
session_name,
...run,
start_time: run.start_time ?? Date.now(),
});
if (this.autoBatchTracing &&
runCreate.trace_id !== undefined &&
runCreate.dotted_order !== undefined) {
void this.processRunOperation({
action: "create",
item: runCreate,
}).catch(console.error);
return;
}
const mergedRunCreateParam = mergeRuntimeEnvIntoRunCreate(runCreate);
const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs`, {
method: "POST",
headers,
body: (0, index_js_2.serialize)(mergedRunCreateParam),
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(response, "create run", true);
}
/**
* Batch ingest/upsert multiple runs in the Langsmith system.
* @param runs
*/
async batchIngestRuns({ runCreates, runUpdates, }) {
if (runCreates === undefined && runUpdates === undefined) {
return;
}
let preparedCreateParams = runCreates?.map((create) => this.prepareRunCreateOrUpdateInputs(create)) ?? [];
let preparedUpdateParams = runUpdates?.map((update) => this.prepareRunCreateOrUpdateInputs(update)) ?? [];
if (preparedCreateParams.length > 0 && preparedUpdateParams.length > 0) {
const createById = preparedCreateParams.reduce((params, run) => {
if (!run.id) {
return params;
}
params[run.id] = run;
return params;
}, {});
const standaloneUpdates = [];
for (const updateParam of preparedUpdateParams) {
if (updateParam.id !== undefined && createById[updateParam.id]) {
createById[updateParam.id] = {
...createById[updateParam.id],
...updateParam,
};
}
else {
standaloneUpdates.push(updateParam);
}
}
preparedCreateParams = Object.values(createById);
preparedUpdateParams = standaloneUpdates;
}
const rawBatch = {
post: preparedCreateParams,
patch: preparedUpdateParams,
};
if (!rawBatch.post.length && !rawBatch.patch.length) {
return;
}
const batchChunks = {
post: [],
patch: [],
};
for (const k of ["post", "patch"]) {
const key = k;
const batchItems = rawBatch[key].reverse();
let batchItem = batchItems.pop();
while (batchItem !== undefined) {
// Type is wrong but this is a deprecated code path anyway
batchChunks[key].push(batchItem);
batchItem = batchItems.pop();
}
}
if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
await this._postBatchIngestRuns((0, index_js_2.serialize)(batchChunks));
}
}
async _postBatchIngestRuns(body) {
const headers = {
...this.headers,
"Content-Type": "application/json",
Accept: "application/json",
};
const response = await this.batchIngestCaller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs/batch`, {
method: "POST",
headers,
body: body,
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(response, "batch create run", true);
}
/**
* Batch ingest/upsert multiple runs in the Langsmith system.
* @param runs
*/
async multipartIngestRuns({ runCreates, runUpdates, }) {
if (runCreates === undefined && runUpdates === undefined) {
return;
}
// transform and convert to dicts
const allAttachments = {};
let preparedCreateParams = [];
for (const create of runCreates ?? []) {
const preparedCreate = this.prepareRunCreateOrUpdateInputs(create);
if (preparedCreate.id !== undefined &&
preparedCreate.attachments !== undefined) {
allAttachments[preparedCreate.id] = preparedCreate.attachments;
}
delete preparedCreate.attachments;
preparedCreateParams.push(preparedCreate);
}
let preparedUpdateParams = [];
for (const update of runUpdates ?? []) {
preparedUpdateParams.push(this.prepareRunCreateOrUpdateInputs(update));
}
// require trace_id and dotted_order
const invalidRunCreate = preparedCreateParams.find((runCreate) => {
return (runCreate.trace_id === undefined || runCreate.dotted_order === undefined);
});
if (invalidRunCreate !== undefined) {
throw new Error(`Multipart ingest requires "trace_id" and "dotted_order" to be set when creating a run`);
}
const invalidRunUpdate = preparedUpdateParams.find((runUpdate) => {
return (runUpdate.trace_id === undefined || runUpdate.dotted_order === undefined);
});
if (invalidRunUpdate !== undefined) {
throw new Error(`Multipart ingest requires "trace_id" and "dotted_order" to be set when updating a run`);
}
// combine post and patch dicts where possible
if (preparedCreateParams.length > 0 && preparedUpdateParams.length > 0) {
const createById = preparedCreateParams.reduce((params, run) => {
if (!run.id) {
return params;
}
params[run.id] = run;
return params;
}, {});
const standaloneUpdates = [];
for (const updateParam of preparedUpdateParams) {
if (updateParam.id !== undefined && createById[updateParam.id]) {
createById[updateParam.id] = {
...createById[updateParam.id],
...updateParam,
};
}
else {
standaloneUpdates.push(updateParam);
}
}
preparedCreateParams = Object.values(createById);
preparedUpdateParams = standaloneUpdates;
}
if (preparedCreateParams.length === 0 &&
preparedUpdateParams.length === 0) {
return;
}
// send the runs in multipart requests
const accumulatedContext = [];
const accumulatedParts = [];
for (const [method, payloads] of [
["post", preparedCreateParams],
["patch", preparedUpdateParams],
]) {
for (const originalPayload of payloads) {
// collect fields to be sent as separate parts
const { inputs, outputs, events, attachments, ...payload } = originalPayload;
const fields = { inputs, outputs, events };
// encode the main run payload
const stringifiedPayload = (0, index_js_2.serialize)(payload);
accumulatedParts.push({
name: `${method}.${payload.id}`,
payload: new Blob([stringifiedPayload], {
type: `application/json; length=${stringifiedPayload.length}`, // encoding=gzip
}),
});
// encode the fields we collected
for (const [key, value] of Object.entries(fields)) {
if (value === undefined) {
continue;
}
const stringifiedValue = (0, index_js_2.serialize)(value);
accumulatedParts.push({
name: `${method}.${payload.id}.${key}`,
payload: new Blob([stringifiedValue], {
type: `application/json; length=${stringifiedValue.length}`,
}),
});
}
// encode the attachments
if (payload.id !== undefined) {
const attachments = allAttachments[payload.id];
if (attachments) {
delete allAttachments[payload.id];
for (const [name, attachment] of Object.entries(attachments)) {
let contentType;
let content;
if (Array.isArray(attachment)) {
[contentType, content] = attachment;
}
else {
contentType = attachment.mimeType;
content = attachment.data;
}
// Validate that the attachment name doesn't contain a '.'
if (name.includes(".")) {
console.warn(`Skipping attachment '${name}' for run ${payload.id}: Invalid attachment name. ` +
`Attachment names must not contain periods ('.'). Please rename the attachment and try again.`);
continue;
}
accumulatedParts.push({
name: `attachment.${payload.id}.${name}`,
payload: new Blob([content], {
type: `${contentType}; length=${content.byteLength}`,
}),
});
}
}
}
// compute context
accumulatedContext.push(`trace=${payload.trace_id},id=${payload.id}`);
}
}
await this._sendMultipartRequest(accumulatedParts, accumulatedContext.join("; "));
}
async _sendMultipartRequest(parts, context) {
try {
// Create multipart form data manually using Blobs
const boundary = "----LangSmithFormBoundary" + Math.random().toString(36).slice(2);
const chunks = [];
for (const part of parts) {
// Add field boundary
chunks.push(new Blob([`--${boundary}\r\n`]));
chunks.push(new Blob([
`Content-Disposition: form-data; name="${part.name}"\r\n`,
`Content-Type: ${part.payload.type}\r\n\r\n`,
]));
chunks.push(part.payload);
chunks.push(new Blob(["\r\n"]));
}
// Add final boundary
chunks.push(new Blob([`--${boundary}--\r\n`]));
// Combine all chunks into a single Blob
const body = new Blob(chunks);
// Convert Blob to ArrayBuffer for compatibility
const arrayBuffer = await body.arrayBuffer();
const res = await this.batchIngestCaller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs/multipart`, {
method: "POST",
headers: {
...this.headers,
"Content-Type": `multipart/form-data; boundary=${boundary}`,
},
body: arrayBuffer,
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(res, "ingest multipart runs", true);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
}
catch (e) {
console.warn(`${e.message.trim()}\n\nContext: ${context}`);
}
}
async updateRun(runId, run) {
(0, _uuid_js_1.assertUuid)(runId);
if (run.inputs) {
run.inputs = this.processInputs(run.inputs);
}
if (run.outputs) {
run.outputs = this.processOutputs(run.outputs);
}
// TODO: Untangle types
const data = { ...run, id: runId };
if (!this._filterForSampling([data], true).length) {
return;
}
if (this.autoBatchTracing &&
data.trace_id !== undefined &&
data.dotted_order !== undefined) {
if (run.end_time !== undefined &&
data.parent_run_id === undefined &&
this.blockOnRootRunFinalization &&
!this.manualFlushMode) {
// Trigger batches as soon as a root trace ends and wait to ensure trace finishes
// in serverless environments.
await this.processRunOperation({ action: "update", item: data }).catch(console.error);
return;
}
else {
void this.processRunOperation({ action: "update", item: data }).catch(console.error);
}
return;
}
const headers = { ...this.headers, "Content-Type": "application/json" };
const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs/${runId}`, {
method: "PATCH",
headers,
body: (0, index_js_2.serialize)(run),
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
await (0, error_js_1.raiseForStatus)(response, "update run", true);
}
async readRun(runId, { loadChildRuns } = { loadChildRuns: false }) {
(0, _uuid_js_1.assertUuid)(runId);
let run = await this._get(`/runs/${runId}`);
if (loadChildRuns && run.child_run_ids) {
run = await this._loadChildRuns(run);
}
return run;
}
async getRunUrl({ runId, run, projectOpts, }) {
if (run !== undefined) {
let sessionId;
if (run.session_id) {
sessionId = run.session_id;
}
else if (projectOpts?.projectName) {
sessionId = (await this.readProject({ projectName: projectOpts?.projectName })).id;
}
else if (projectOpts?.projectId) {
sessionId = projectOpts?.projectId;
}
else {
const project = await this.readProject({
projectName: (0, env_js_1.getLangSmithEnvironmentVariable)("PROJECT") || "default",
});
sessionId = project.id;
}
const tenantId = await this._getTenantId();
return `${this.getHostUrl()}/o/${tenantId}/projects/p/${sessionId}/r/${run.id}?poll=true`;
}
else if (runId !== undefined) {
const run_ = await this.readRun(runId);
if (!run_.app_path) {
throw new Error(`Run ${runId} has no app_path`);
}
const baseUrl = this.getHostUrl();
return `${baseUrl}${run_.app_path}`;
}
else {
throw new Error("Must provide either runId or run");
}
}
async _loadChildRuns(run) {
const childRuns = await toArray(this.listRuns({ id: run.child_run_ids }));
const treemap = {};
const runs = {};
// TODO: make dotted order required when the migration finishes
childRuns.sort((a, b) => (a?.dotted_order ?? "").localeCompare(b?.dotted_order ?? ""));
for (const childRun of childRuns) {
if (childRun.parent_run_id === null ||
childRun.parent_run_id === undefined) {
throw new Error(`Child run ${childRun.id} has no parent`);
}
if (!(childRun.parent_run_id in treemap)) {
treemap[childRun.parent_run_id] = [];
}
treemap[childRun.parent_run_id].push(childRun);
runs[childRun.id] = childRun;
}
run.child_runs = treemap[run.id] || [];
for (const runId in treemap) {
if (runId !== run.id) {
runs[runId].child_runs = treemap[runId];
}
}
return run;
}
/**
* List runs from the LangSmith server.
* @param projectId - The ID of the project to filter by.
* @param projectName - The name of the project to filter by.
* @param parentRunId - The ID of the parent run to filter by.
* @param traceId - The ID of the trace to filter by.
* @param referenceExampleId - The ID of the reference example to filter by.
* @param startTime - The start time to filter by.
* @param isRoot - Indicates whether to only return root runs.
* @param runType - The run type to filter by.
* @param error - Indicates whether to filter by error runs.
* @param id - The ID of the run to filter by.
* @param query - The query string to filter by.
* @param filter - The filter string to apply to the run spans.
* @param traceFilter - The filter string to apply on the root run of the trace.
* @param treeFilter - The filter string to apply on other runs in the trace.
* @param limit - The maximum number of runs to retrieve.
* @returns {AsyncIterable<Run>} - The runs.
*
* @example
* // List all runs in a project
* const projectRuns = client.listRuns({ projectName: "<your_project>" });
*
* @example
* // List LLM and Chat runs in the last 24 hours
* const todaysLLMRuns = client.listRuns({
* projectName: "<your_project>",
* start_time: new Date(Date.now() - 24 * 60 * 60 * 1000),
* run_type: "llm",
* });
*
* @example
* // List traces in a project
* const rootRuns = client.listRuns({
* projectName: "<your_project>",
* execution_order: 1,
* });
*
* @example
* // List runs without errors
* const correctRuns = client.listRuns({
* projectName: "<your_project>",
* error: false,
* });
*
* @example
* // List runs by run ID
* const runIds = [
* "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836",
* "9398e6be-964f-4aa4-8ae9-ad78cd4b7074",
* ];
* const selectedRuns = client.listRuns({ run_ids: runIds });
*
* @example
* // List all "chain" type runs that took more than 10 seconds and had `total_tokens` greater than 5000
* const chainRuns = client.listRuns({
* projectName: "<your_project>",
* filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))',
* });
*
* @example
* // List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1
* const goodExtractorRuns = client.listRuns({
* projectName: "<your_project>",
* filter: 'eq(name, "extractor")',
* traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
* });
*
* @example
* // List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0
* const complexRuns = client.listRuns({
* projectName: "<your_project>",
* filter: 'and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))',
* });
*
* @example
* // List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds
* const taggedRuns = client.listRuns({
* projectName: "<your_project>",
* filter: 'and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))',
* });
*/
async *listRuns(props) {
const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, isRoot, runType, error, id, query, filter, traceFilter, treeFilter, limit, select, } = props;
let projectIds = [];
if (projectId) {
projectIds = Array.isArray(projectId) ? projectId : [projectId];
}
if (projectName) {
const projectNames = Array.isArray(projectName)
? projectName
: [projectName];
const projectIds_ = await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)));
projectIds.push(...projectIds_);
}
const default_select = [
"app_path",
"child_run_ids",
"completion_cost",
"completion_tokens",
"dotted_order",
"end_time",
"error",
"events",
"extra",
"feedback_stats",
"first_token_time",
"id",
"inputs",
"name",
"outputs",
"parent_run_id",
"parent_run_ids",
"prompt_cost",
"prompt_tokens",
"reference_example_id",
"run_type",
"session_id",
"start_time",
"status",
"tags",
"total_cost",
"total_tokens",
"trace_id",
];
const body = {
session: projectIds.length ? projectIds : null,
run_type: runType,
reference_example: referenceExampleId,
query,
filter,
trace_filter: traceFilter,
tree_filter: treeFilter,
execution_order: executionOrder,
parent_run: parentRunId,
start_time: startTime ? startTime.toISOString() : null,
error,
id,
limit,
trace: traceId,
select: select ? select : default_select,
is_root: isRoot,
};
let runsYielded = 0;
for await (const runs of this._getCursorPaginatedList("/runs/query", body)) {
if (limit) {
if (runsYielded >= limit) {
break;
}
if (runs.length + runsYielded > limit) {
const newRuns = runs.slice(0, limit - runsYielded);
yield* newRuns;
break;
}
runsYielded += runs.length;
yield* runs;
}
else {
yield* runs;
}
}
}
async getRunStats({ id, trace, parentRun, runType, projectNames, projectIds, referenceExampleIds, startTime, endTime, error, query, filter, traceFilter, treeFilter, isRoot, dataSourceType, }) {
let projectIds_ = projectIds || [];
if (projectNames) {
projectIds_ = [
...(projectIds || []),
...(await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)))),
];
}
const payload = {
id,
trace,
parent_run: parentRun,
run_type: runType,
session: projectIds_,
reference_example: referenceExampleIds,
start_time: startTime,
end_time: endTime,
error,
query,
filter,
trace_filter: traceFilter,
tree_filter: treeFilter,
is_root: isRoot,
data_source_type: dataSourceType,
};
// Remove undefined values from the payload
const filteredPayload = Object.fromEntries(Object.entries(payload).filter(([_, value]) => value !== undefined));
const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs/stats`, {
method: "POST",
headers: this.headers,
body: JSON.stringify(filteredPayload),
signal: AbortSignal.timeout(this.timeout_ms),
...this.fetchOptions,
});
const result = await response.json();
return result;
}
async shareRun(runId, { shareId } = {}) {
const data = {
run_id: runId,
share_token: shareId || uuid.v4(),
};
(0, _uuid_js_1.