dtamind-components
Version:
Apps integration for Dtamind. Contain Nodes and Credentials.
229 lines • 10.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var _a;
Object.defineProperty(exports, "__esModule", { value: true });
exports.EvaluationRunner = void 0;
const axios_1 = __importDefault(require("axios"));
const uuid_1 = require("uuid");
const modelLoader_1 = require("../src/modelLoader");
class EvaluationRunner {
static async getAndDeleteMetrics(id) {
const val = _a.metrics.get(id);
if (val) {
try {
//first lets get the provider and model
let selectedModel = undefined;
let selectedProvider = undefined;
if (val && val.length > 0) {
let modelName = '';
let providerName = '';
for (let i = 0; i < val.length; i++) {
const metric = val[i];
if (typeof metric === 'object') {
modelName = metric['model'];
providerName = metric['provider'];
}
else {
modelName = JSON.parse(metric)['model'];
providerName = JSON.parse(metric)['provider'];
}
if (modelName) {
selectedModel = modelName;
}
if (providerName) {
selectedProvider = providerName;
}
}
}
if (selectedProvider && selectedModel) {
const modelConfig = await _a.getCostMetrics(selectedProvider, selectedModel);
if (modelConfig) {
val.push(JSON.stringify({ cost_values: modelConfig }));
}
}
}
catch (error) {
//stay silent
}
}
_a.metrics.delete(id);
return val;
}
static addMetrics(id, metric) {
if (_a.metrics.has(id)) {
_a.metrics.get(id)?.push(metric);
}
else {
_a.metrics.set(id, [metric]);
}
}
constructor(baseURL) {
this.baseURL = '';
this.baseURL = baseURL;
}
getChatflowApiKey(chatflowId, apiKeys = []) {
return apiKeys.find((item) => item.chatflowId === chatflowId)?.apiKey || '';
}
async runEvaluations(data) {
const chatflowIds = JSON.parse(data.chatflowId);
const returnData = {};
returnData.evaluationId = data.evaluationId;
returnData.runDate = new Date();
returnData.rows = [];
for (let i = 0; i < data.dataset.rows.length; i++) {
returnData.rows.push({
input: data.dataset.rows[i].input,
expectedOutput: data.dataset.rows[i].output,
itemNo: data.dataset.rows[i].sequenceNo,
evaluations: [],
status: 'pending'
});
}
for (let i = 0; i < chatflowIds.length; i++) {
const chatflowId = chatflowIds[i];
await this.evaluateChatflow(chatflowId, this.getChatflowApiKey(chatflowId, data.apiKeys), data, returnData);
}
return returnData;
}
async evaluateChatflow(chatflowId, apiKey, data, returnData) {
for (let i = 0; i < data.dataset.rows.length; i++) {
const item = data.dataset.rows[i];
const uuid = (0, uuid_1.v4)();
const headers = {
'X-Request-ID': uuid,
'X-Dtamind-Evaluation': 'true'
};
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
}
let axiosConfig = {
headers: headers
};
let startTime = performance.now();
const runData = {};
runData.chatflowId = chatflowId;
runData.startTime = startTime;
const postData = { question: item.input, evaluationRunId: uuid, evaluation: true };
if (data.sessionId) {
postData.overrideConfig = { sessionId: data.sessionId };
}
try {
let response = await axios_1.default.post(`${this.baseURL}/api/v1/prediction/${chatflowId}`, postData, axiosConfig);
let agentFlowMetrics = [];
if (response?.data?.agentFlowExecutedData) {
for (let i = 0; i < response.data.agentFlowExecutedData.length; i++) {
const agentFlowExecutedData = response.data.agentFlowExecutedData[i];
const input_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.input_tokens || 0;
const output_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.output_tokens || 0;
const total_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.total_tokens || input_tokens + output_tokens;
const metrics = {
promptTokens: input_tokens,
completionTokens: output_tokens,
totalTokens: total_tokens,
provider: agentFlowExecutedData.data?.input?.llmModelConfig?.llmModel ||
agentFlowExecutedData.data?.input?.agentModelConfig?.agentModel,
model: agentFlowExecutedData.data?.input?.llmModelConfig?.modelName ||
agentFlowExecutedData.data?.input?.agentModelConfig?.modelName,
nodeLabel: agentFlowExecutedData?.nodeLabel,
nodeId: agentFlowExecutedData?.nodeId
};
if (metrics.provider && metrics.model) {
const modelConfig = await _a.getCostMetrics(metrics.provider, metrics.model);
if (modelConfig) {
metrics.cost_values = {
input_cost: (modelConfig.cost_values.input_cost || 0) * (input_tokens / 1000),
output_cost: (modelConfig.cost_values.output_cost || 0) * (output_tokens / 1000)
};
metrics.cost_values.total_cost = metrics.cost_values.input_cost + metrics.cost_values.output_cost;
}
}
agentFlowMetrics.push(metrics);
}
}
const endTime = performance.now();
const timeTaken = (endTime - startTime).toFixed(2);
if (response?.data?.metrics) {
runData.metrics = response.data.metrics;
runData.metrics.push({
apiLatency: timeTaken
});
}
else {
runData.metrics = [
{
apiLatency: timeTaken
}
];
}
if (agentFlowMetrics.length > 0) {
runData.nested_metrics = agentFlowMetrics;
}
runData.status = 'complete';
let resultText = '';
if (response.data.text)
resultText = response.data.text;
else if (response.data.json)
resultText = '```json\n' + JSON.stringify(response.data.json, null, 2);
else
resultText = JSON.stringify(response.data, null, 2);
runData.actualOutput = resultText;
runData.latency = timeTaken;
runData.error = '';
}
catch (error) {
runData.status = 'error';
runData.actualOutput = '';
runData.error = error?.response?.data?.message
? error.response.data.message
: error?.message
? error.message
: 'Unknown error';
try {
if (runData.error.indexOf('-') > -1) {
// if there is a dash, remove all content before
runData.error = 'Error: ' + runData.error.substr(runData.error.indexOf('-') + 1).trim();
}
}
catch (error) {
//stay silent
}
const endTime = performance.now();
const timeTaken = (endTime - startTime).toFixed(2);
runData.metrics = [
{
apiLatency: timeTaken
}
];
runData.latency = timeTaken;
}
runData.uuid = uuid;
returnData.rows[i].evaluations.push(runData);
}
return returnData;
}
}
exports.EvaluationRunner = EvaluationRunner;
_a = EvaluationRunner;
EvaluationRunner.metrics = new Map();
EvaluationRunner.getCostMetrics = async (selectedProvider, selectedModel) => {
let modelConfig = await (0, modelLoader_1.getModelConfigByModelName)(modelLoader_1.MODEL_TYPE.CHAT, selectedProvider, selectedModel);
if (modelConfig) {
if (modelConfig['cost_values']) {
return modelConfig.cost_values;
}
return { cost_values: modelConfig };
}
else {
modelConfig = await (0, modelLoader_1.getModelConfigByModelName)(modelLoader_1.MODEL_TYPE.LLM, selectedProvider, selectedModel);
if (modelConfig) {
if (modelConfig['cost_values']) {
return modelConfig.cost_values;
}
return { cost_values: modelConfig };
}
}
return undefined;
};
//# sourceMappingURL=EvaluationRunner.js.map