parea-ai
Version:
Client SDK library to connect to Parea AI.
81 lines (80 loc) • 4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Trial = void 0;
const types_1 = require("./types");
const TraceManager_1 = require("../utils/core/TraceManager");
const experimentContext_1 = require("./experimentContext");
const types_2 = require("../types");
/**
* Represents a single trial in an experiment.
*/
class Trial {
/**
* Creates a new Trial instance.
* @param data The input data for the trial.
* @param func The function to be executed for the trial.
* @param experimentUUID The UUID of the experiment this trial belongs to.
* @param maxRetries - The maximum number of retries to wait for eval to finish. Each retry waits for 1s. Default is 60.
*/
constructor(data, func, experimentUUID, maxRetries) {
this.data = data;
this.func = func;
this.experimentUUID = experimentUUID;
this.maxRetries = maxRetries;
this.state = types_2.ExperimentStatus.PENDING;
}
/**
* Runs the trial and returns the result.
* @returns A promise that resolves to the trial result.
*/
async run() {
this.state = types_2.ExperimentStatus.RUNNING;
const traceManager = TraceManager_1.TraceManager.getInstance();
return experimentContext_1.experimentContext.runInContext(this.experimentUUID, async () => {
try {
const result = await traceManager.runInContext(async () => {
process.env.PAREA_OS_ENV_EXPERIMENT_UUID = this.experimentUUID;
const { target, ...dataInput } = this.data;
const dataSamples = Object.values(dataInput);
let funcResult;
if (target !== undefined) {
const _target = typeof target === 'object' ? JSON.stringify(target) : target;
funcResult = await this.func(...dataSamples, _target);
}
else {
funcResult = await this.func(...dataSamples);
}
return funcResult;
});
const { state, error } = await this.waitForLogs();
this.state = state;
const scores = experimentContext_1.experimentContext.getScores(this.experimentUUID);
const logs = experimentContext_1.experimentContext.getLogs(this.experimentUUID);
return new types_1.TrialResult(this.data, result, error || null, state, scores, logs);
}
catch (error) {
this.state = types_2.ExperimentStatus.FAILED;
const e = error instanceof Error ? error : new Error(String(error));
return new types_1.TrialResult(this.data, null, e, this.state, null, null);
}
});
}
async waitForLogs() {
await new Promise((resolve) => setTimeout(resolve, 2500)); // Wait for 2.5s before checking logs
for (let i = 1; i < this.maxRetries; i++) {
const logs = experimentContext_1.experimentContext.getLogs(this.experimentUUID);
if (logs.length > 0) {
return { state: types_2.ExperimentStatus.COMPLETED };
}
// log every 10 retries
if (i % 10 === 0) {
console.debug(`Waiting for eval to finish for trial in experiment ${this.experimentUUID}. Retrying (${i}/${this.maxRetries})...`);
}
await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1s before checking again
}
const msg = `No logs were collected for trial in experiment ${this.experimentUUID} after ${this.maxRetries} trys. Eval function likely did not finish, try increasing maxRetries on p.experiment. e.g: p.experiment('ExperimentName', data, func, { maxRetries: 120 })`;
console.warn(msg);
return { state: types_2.ExperimentStatus.FAILED, error: new Error(msg) };
}
}
exports.Trial = Trial;