n8n/dist/modules/agent-evals/agent-eval.service.js

n8n Workflow Automation Tool

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AgentEvalService = void 0;
const backend_common_1 = require("@n8n/backend-common");
const db_1 = require("@n8n/db");
const di_1 = require("@n8n/di");
const bad_request_error_1 = require("../../errors/response-errors/bad-request.error");
const not_found_error_1 = require("../../errors/response-errors/not-found.error");
const agent_repository_1 = require("../../modules/agents/repositories/agent.repository");
const agent_eval_case_generation_service_1 = require("./agent-eval-case-generation.service");
const agent_eval_record_mappers_1 = require("./agent-eval-record-mappers");
const agent_eval_runner_service_1 = require("./agent-eval-runner.service");
const agent_evals_required_modules_1 = require("./agent-evals-required-modules");
const CANCELLABLE_STATUSES = new Set(['new', 'running']);
let AgentEvalService = class AgentEvalService {
    constructor(moduleRegistry, agentRepository, datasetRepository, runRepository, resultRepository, runner, caseGenerationService) {
        this.moduleRegistry = moduleRegistry;
        this.agentRepository = agentRepository;
        this.datasetRepository = datasetRepository;
        this.runRepository = runRepository;
        this.resultRepository = resultRepository;
        this.runner = runner;
        this.caseGenerationService = caseGenerationService;
    }
    async listDatasets(agentId, projectId) {
        await this.assertAgentInProject(agentId, projectId);
        const datasets = await this.datasetRepository.findByAgentId(agentId);
        return datasets.map(agent_eval_record_mappers_1.toDatasetRecord);
    }
    async getDataset(agentId, projectId, datasetId) {
        await this.assertAgentInProject(agentId, projectId);
        return (0, agent_eval_record_mappers_1.toDatasetRecord)(await this.resolveDataset(agentId, datasetId));
    }
    async createDataset(user, agentId, projectId, payload) {
        await this.assertAgentInProject(agentId, projectId);
        if (payload.agentId !== agentId) {
            throw new bad_request_error_1.BadRequestError(`The dataset's agentId ('${payload.agentId}') does not match the agent in the URL.`);
        }
        const dataset = await this.datasetRepository.createDataset({
            name: payload.name,
            description: payload.description ?? null,
            agentId,
            datasetSource: payload.datasetSource,
            datasetRef: payload.datasetRef,
            columnMapping: payload.columnMapping ?? null,
            createdById: user.id,
        });
        return (0, agent_eval_record_mappers_1.toDatasetRecord)(dataset);
    }
    async updateDataset(agentId, projectId, datasetId, payload) {
        await this.assertAgentInProject(agentId, projectId);
        const updated = await this.datasetRepository.updateDataset(datasetId, agentId, payload);
        if (!updated)
            throw new not_found_error_1.NotFoundError(`Agent eval dataset ${datasetId} not found.`);
        return (0, agent_eval_record_mappers_1.toDatasetRecord)(updated);
    }
    async deleteDataset(agentId, projectId, datasetId) {
        await this.assertAgentInProject(agentId, projectId);
        const deleted = await this.datasetRepository.deleteDataset(datasetId, agentId);
        if (!deleted)
            throw new not_found_error_1.NotFoundError(`Agent eval dataset ${datasetId} not found.`);
    }
    async generateDraftCases(user, agentId, projectId, options) {
        await this.assertAgentInProject(agentId, projectId);
        return await this.caseGenerationService.generateDraftCases(user, projectId, agentId, options);
    }
    async startRun(user, agentId, projectId, datasetId, payload) {
        await this.assertAgentInProject(agentId, projectId);
        await this.resolveDataset(agentId, datasetId);
        if (payload.agentVersionId !== undefined) {
            throw new bad_request_error_1.BadRequestError('Pinning an agent version for an eval run is not supported yet.');
        }
        const { runId } = await this.runner.startRun(datasetId, projectId, user);
        const run = await this.runRepository.findById(runId);
        if (!run)
            throw new not_found_error_1.NotFoundError(`Agent eval run ${runId} not found.`);
        return (0, agent_eval_record_mappers_1.toRunRecord)(run);
    }
    async listRuns(agentId, projectId, datasetId, page) {
        await this.assertAgentInProject(agentId, projectId);
        await this.resolveDataset(agentId, datasetId);
        const [runs, count] = await this.runRepository.findAndCountByDatasetIdAndAgentId(datasetId, agentId, page);
        return { count, data: runs.map(agent_eval_record_mappers_1.toRunRecord) };
    }
    async getRunDetail(agentId, projectId, runId, page) {
        await this.assertAgentInProject(agentId, projectId);
        const run = await this.resolveRun(agentId, runId);
        const [results, count] = await this.resultRepository.findAndCountByRunId(runId, page);
        return { ...(0, agent_eval_record_mappers_1.toRunRecord)(run), results: { count, data: results.map(agent_eval_record_mappers_1.toResultRecord) } };
    }
    async getRunSummary(agentId, projectId, runId) {
        await this.assertAgentInProject(agentId, projectId);
        return await this.runner.getRunSummary(runId, agentId);
    }
    async cancelRun(agentId, projectId, runId) {
        await this.assertAgentInProject(agentId, projectId);
        const run = await this.resolveRun(agentId, runId);
        if (!CANCELLABLE_STATUSES.has(run.status)) {
            throw new bad_request_error_1.BadRequestError(`Agent eval run ${runId} has already finished ('${run.status}').`);
        }
        await this.runRepository.requestCancellation(runId);
        const updated = await this.runRepository.findById(runId);
        return (0, agent_eval_record_mappers_1.toRunRecord)(updated ?? run);
    }
    async assertAgentInProject(agentId, projectId) {
        (0, agent_evals_required_modules_1.assertRequiredModulesActive)(this.moduleRegistry);
        const agent = await this.agentRepository.findByIdAndProjectId(agentId, projectId);
        if (!agent)
            throw new not_found_error_1.NotFoundError(`Agent ${agentId} not found.`);
    }
    async resolveDataset(agentId, datasetId) {
        const dataset = await this.datasetRepository.findByIdAndAgentId(datasetId, agentId);
        if (!dataset)
            throw new not_found_error_1.NotFoundError(`Agent eval dataset ${datasetId} not found.`);
        return dataset;
    }
    async resolveRun(agentId, runId) {
        const run = await this.runRepository.findByIdAndAgentId(runId, agentId);
        if (!run)
            throw new not_found_error_1.NotFoundError(`Agent eval run ${runId} not found.`);
        return run;
    }
};
exports.AgentEvalService = AgentEvalService;
exports.AgentEvalService = AgentEvalService = __decorate([
    (0, di_1.Service)(),
    __metadata("design:paramtypes", [backend_common_1.ModuleRegistry, agent_repository_1.AgentRepository, db_1.AgentEvalDatasetRepository, db_1.AgentEvalRunRepository, db_1.AgentEvalResultRepository, agent_eval_runner_service_1.AgentEvalRunnerService, agent_eval_case_generation_service_1.AgentEvalCaseGenerationService])
], AgentEvalService);
//# sourceMappingURL=agent-eval.service.js.map