@maximai/maxim-js
Version:
Maxim AI JS SDK. Visit https://getmaxim.ai for more info.
748 lines • 27.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.MaximTestRunAPI = void 0;
const dataset_1 = require("../models/dataset");
const platform_1 = require("../platform");
const maxim_1 = require("./maxim");
class MaximTestRunAPI extends maxim_1.MaximAPI {
constructor(baseUrl, apiKey, isDebug) {
super(baseUrl, apiKey, isDebug);
}
async createTestRun(name, workspaceId, runType, evaluatorConfig, requiresLocalRun, workflowId, promptVersionId, promptChainVersionId, humanEvaluationConfig, tags, simulationConfig, connectedRepoId) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/create`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
name,
workspaceId,
runType,
evaluatorConfig,
requiresLocalRun,
workflowId,
promptVersionId,
promptChainVersionId,
humanEvaluationConfig,
tags,
simulationConfig,
connectedRepoId,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async markTestRunFailed(testRunId) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/mark-failed`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunId,
}),
})
.then((response) => {
if (response.error) {
reject(response.error);
}
else {
resolve();
}
})
.catch((error) => {
reject(error);
});
});
}
async updateSimulationStatus(testRunEntryId, status) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/simulation/update-status`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunEntryId,
status,
}),
})
.then((response) => {
if (response.error) {
reject(response.error);
}
else {
resolve();
}
})
.catch((error) => {
reject(error);
});
});
}
async attachDatasetToTestRun(testRunId, datasetId) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/attach-dataset`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunId,
datasetId,
}),
})
.then((response) => {
if (response.error) {
reject(response.error);
}
else {
resolve();
}
})
.catch((error) => {
reject(error);
});
});
}
/**
* Checks if a value is already in Variable format.
*/
isVariable(value) {
return (typeof value === "object" &&
value !== null &&
"type" in value &&
"payload" in value &&
Object.values(dataset_1.VariableType).includes(value.type));
}
/**
* Converts dataEntry values from string/string[] to Variable type format.
* - string -> { type: "text", payload: string }
* - string[] -> { type: "file", payload: UrlAttachment[] }
* - null/undefined -> undefined (skipped)
*/
convertDataEntryToVariables(dataEntry) {
const result = {};
for (const [key, value] of Object.entries(dataEntry)) {
if (value === null || value === undefined) {
// Skip null/undefined values
continue;
}
if (Array.isArray(value)) {
// Convert string array to FILE Variable with UrlAttachment[]
const attachments = value.map((url, index) => ({
type: "url",
id: `${key}-${index}`,
url: url,
}));
result[key] = {
type: dataset_1.VariableType.FILE,
payload: attachments,
};
}
else {
// Convert string to TEXT Variable
result[key] = {
type: dataset_1.VariableType.TEXT,
payload: value,
};
}
}
return result;
}
/**
* Normalizes raw dataEntry to Variable format. If any value is a plain string/array,
* converts via convertDataEntryToVariables; otherwise returns the entry as-is (already Variable).
*/
normalizeDataEntryToVariables(rawDataEntry) {
const needsConversion = Object.values(rawDataEntry).some((value) => value !== null && value !== undefined && !this.isVariable(value));
return needsConversion
? this.convertDataEntryToVariables(rawDataEntry)
: rawDataEntry;
}
/**
* Signed upload URL for log-repository attachments (same service as {@link MaximAttachmentAPI}).
*/
async getLogAttachmentUploadUrl(key, mimeType, size) {
const response = await this.fetch(`/api/sdk/v1/log-repositories/attachments/upload-url?key=${encodeURIComponent(key)}&mimeType=${encodeURIComponent(mimeType)}&size=${size}`);
if ("error" in response) {
throw response.error;
}
return response.data;
}
async uploadBufferToSignedUrl(url, data, mimeType) {
const response = await this.axiosInstance.put(url, data, {
headers: {
"Content-Type": mimeType,
"Content-Length": data.length.toString(),
},
responseType: "text",
timeout: 120000,
transformRequest: [(body) => body],
transformResponse: [(body) => body],
baseURL: "",
});
if (response.status >= 200 && response.status < 300) {
return;
}
if (response.data && typeof response.data === "object" && "error" in response.data) {
throw response.data.error;
}
throw response.data;
}
async readFileOrFileDataAttachment(attachment) {
var _a;
const maxFileSizeBytes = 1024 * 1024 * 100;
if (attachment.type === "fileData") {
const fileData = attachment.data;
const mimeType = attachment.mimeType || "application/octet-stream";
const size = fileData.length;
if (size > maxFileSizeBytes) {
throw new Error(`File size exceeds the maximum allowed size of ${maxFileSizeBytes} bytes`);
}
return { fileData, mimeType, size };
}
if (!platform_1.platform.features.fileIoSupported) {
throw new Error("File operations are not supported in this environment");
}
let stats;
try {
stats = await platform_1.platform.fs.readFile(attachment.path);
}
catch {
throw new Error(`File not found: ${attachment.path}`);
}
if (stats.data.length > maxFileSizeBytes) {
throw new Error(`File size exceeds the maximum allowed size of ${maxFileSizeBytes} bytes`);
}
let fileData;
try {
fileData = Buffer.from(stats.data);
}
catch {
throw new Error(`File not found: ${attachment.path}`);
}
let mimeType = attachment.mimeType || "application/octet-stream";
if (!mimeType || mimeType === "application/octet-stream") {
const source = (_a = attachment.name) !== null && _a !== void 0 ? _a : attachment.path;
const inferred = platform_1.platform.mime.lookup(source);
if (inferred) {
mimeType = inferred;
}
}
return { fileData, mimeType, size: fileData.length };
}
/**
* Resolves a remotely fetchable URL for push payloads: URL attachments pass through;
* local file / in-memory fileData attachments are uploaded via the log attachment pipeline.
*/
async resolveAttachmentUrlForPush(attachment, testRunId) {
var _a;
if (attachment.type === "url") {
const u = attachment.url;
if (!u || (!u.startsWith("http://") && !u.startsWith("https://"))) {
throw new Error(`Invalid URL: ${u}`);
}
return u;
}
const { fileData, mimeType, size } = await this.readFileOrFileDataAttachment(attachment);
const key = `test-run/${testRunId}/${attachment.id}`;
const { url } = await this.getLogAttachmentUploadUrl(key, mimeType, size);
try {
await this.uploadBufferToSignedUrl(url, fileData, mimeType);
}
catch (error) {
const name = (_a = attachment.name) !== null && _a !== void 0 ? _a : attachment.id;
throw new Error(`Failed to upload attachment ${name}: ${error instanceof Error ? error.message : String(error)}`);
}
return url;
}
/**
* Converts Variable format to API format for dataEntry (TEXT / JSON only).
*/
convertNonFileVariableToAPIFormat(variable) {
if (variable.type === dataset_1.VariableType.TEXT || variable.type === dataset_1.VariableType.JSON) {
return {
type: "text",
payload: variable.payload,
};
}
return {
type: "text",
payload: "",
};
}
/**
* Converts a FILE variable to API format, uploading local/fileData attachments first.
*/
async convertFileVariableToAPIFormat(variable, testRunId) {
const files = await Promise.all(variable.payload.map(async (attachment) => {
const url = await this.resolveAttachmentUrlForPush(attachment, testRunId);
return {
id: attachment.id,
url,
name: attachment.name,
type: attachment.mimeType || "application/octet-stream",
};
}));
return {
type: "file",
payload: { files },
};
}
/**
* Converts dataEntry from Variable format to API format.
* FILE variables upload non-URL attachments before building the payload.
*/
async convertDataEntryToAPIFormat(dataEntry, testRunId) {
const result = {};
for (const [key, value] of Object.entries(dataEntry)) {
if (value === null || value === undefined) {
result[key] = value;
continue;
}
if (this.isVariable(value)) {
if (value.type === dataset_1.VariableType.FILE) {
result[key] = await this.convertFileVariableToAPIFormat(value, testRunId);
}
else {
result[key] = this.convertNonFileVariableToAPIFormat(value);
}
}
else if (typeof value === "string") {
result[key] = {
type: "text",
payload: value,
};
}
else if (Array.isArray(value)) {
const files = value.map((url, index) => ({
id: `${key}-${index}`,
url: url,
type: "application/octet-stream",
}));
result[key] = {
type: "file",
payload: {
files,
},
};
}
}
return result;
}
async createTestRunEntry({ testRun, }) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/test-run-entry/create`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRun,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async pushTestRunEntry({ testRun, runConfig, entry, localSimulation }) {
const convertedEntry = entry.dataEntry
? {
...entry,
dataEntry: await this.convertDataEntryToAPIFormat(entry.dataEntry, testRun.id),
}
: entry;
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v4/test-run/push`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRun,
runConfig,
entry: convertedEntry,
...(localSimulation !== undefined && { localSimulation }),
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve();
}
})
.catch((error) => {
reject(error);
});
});
}
async markTestRunProcessed(testRunId) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/mark-processed`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunId,
}),
})
.then((response) => {
if (response.error) {
reject(response.error);
}
else {
resolve();
}
})
.catch((error) => {
reject(error);
});
});
}
async getTestRunStatus(testRunId) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/status?testRunId=${testRunId}`)
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async getTestRunFinalResult(testRunId) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/result?testRunId=${testRunId}`)
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async executeWorkflowForData({ dataEntry, workflowId, contextToEvaluate, }) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/execute/workflow`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
workflowId,
dataEntry,
contextToEvaluate,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async executePromptForData({ promptVersionId, input, dataEntry, contextToEvaluate, simulationConfig, }) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/execute/prompt`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
promptVersionId,
input,
dataEntry,
contextToEvaluate,
simulationConfig
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async executePromptChainForData({ promptChainVersionId, input, dataEntry, contextToEvaluate, }) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v1/test-run/execute/prompt-chain`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
promptChainVersionId,
input,
dataEntry,
contextToEvaluate,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async executeSimulationPromptForData({ testRunId, promptVersionId, workspaceId, datasetEntryId, entry, simulationConfig, }) {
const convertedEntry = (entry === null || entry === void 0 ? void 0 : entry.dataEntry) != null
? {
...entry,
dataEntry: this.normalizeDataEntryToVariables(entry.dataEntry),
}
: entry;
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/execute/simulation/prompt`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunId,
promptVersionId,
workspaceId,
datasetEntryId,
entry: convertedEntry,
simulationConfig,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async getSimulationPromptStatus({ workspaceId, testRunEntryId, }) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/execute/simulation/prompt?workspaceId=${workspaceId}&testRunEntryId=${testRunEntryId}`, {
method: "GET",
headers: {
Accept: "application/json",
},
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async executeSimulationWorkflowForData({ testRunId, workflowId, workspaceId, datasetEntryId, entry, simulationConfig, }) {
const convertedEntry = (entry === null || entry === void 0 ? void 0 : entry.dataEntry) != null
? {
...entry,
dataEntry: this.normalizeDataEntryToVariables(entry.dataEntry),
}
: entry;
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/execute/simulation/workflow`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunId,
workflowId,
workspaceId,
datasetEntryId,
entry: convertedEntry,
simulationConfig,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async getSimulationWorkflowStatus({ workspaceId, testRunEntryId, }) {
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/execute/simulation/workflow?workspaceId=${workspaceId}&testRunEntryId=${testRunEntryId}`, {
method: "GET",
headers: {
Accept: "application/json",
},
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
resolve(response.data);
}
})
.catch((error) => {
reject(error);
});
});
}
async executeSimulationLocalExecution({ testRunId, workspaceId, datasetEntryId, entry, simulationConfig, conversationHistory, testRunEntryId, }) {
const convertedEntry = (entry === null || entry === void 0 ? void 0 : entry.dataEntry) != null
? {
...entry,
dataEntry: this.normalizeDataEntryToVariables(entry.dataEntry),
}
: entry;
const serializedSimulationConfig = this.serializeSimulationConfig(simulationConfig);
return new Promise((resolve, reject) => {
this.fetch(`/api/sdk/v2/test-run/simulation/local-execution`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify({
testRunId,
workspaceId,
datasetEntryId,
entry: convertedEntry,
simulationConfig: serializedSimulationConfig,
conversationHistory,
testRunEntryId,
}),
})
.then((response) => {
if ("error" in response) {
reject(response.error);
}
else {
// Normalize userInput: backend returns string|null,
// convert to {input: string} for consumer convenience
const normalizedData = {
...response.data,
userInput: this.normalizeUserInput(response.data.userInput),
};
resolve(normalizedData);
}
})
.catch((error) => {
reject(error);
});
});
}
/**
* Normalize userInput from backend (string|null) to Record<string, unknown>|null.
* Backend returns plain string; SDK consumers expect {input: string}.
*/
normalizeUserInput(userInput) {
if (userInput === null || userInput === undefined)
return null;
if (typeof userInput === "string")
return { input: userInput };
if (typeof userInput === "object")
return userInput;
return { input: String(userInput) };
}
/**
* Serialize simulationConfig for the backend API.
* Flattens customSimulator fields to top level (matching backend's flat schema).
*/
serializeSimulationConfig(config) {
if (!config)
return undefined;
const result = { ...config };
if (config.customSimulator) {
result["type"] = "CUSTOM";
result["simulatorPrompt"] = config.customSimulator.simulatorPrompt;
if (config.customSimulator.model)
result["model"] = config.customSimulator.model;
if (config.customSimulator.provider)
result["provider"] = config.customSimulator.provider;
if (config.customSimulator.variables)
result["variables"] = config.customSimulator.variables;
if (config.customSimulator.variableBindings)
result["variableBindings"] = config.customSimulator.variableBindings;
if (config.customSimulator.modelParameters)
result["modelParameters"] = config.customSimulator.modelParameters;
delete result["customSimulator"];
}
return result;
}
}
exports.MaximTestRunAPI = MaximTestRunAPI;
//# sourceMappingURL=testRun.js.map