n8n/dist/evaluation.ee/evaluation-collection.service.js

n8n Workflow Automation Tool

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.EvaluationCollectionService = void 0;
const db_1 = require("@n8n/db");
const di_1 = require("@n8n/di");
const typeorm_1 = require("@n8n/typeorm");
const nanoid_1 = require("nanoid");
const bad_request_error_1 = require("../errors/response-errors/bad-request.error");
const not_found_error_1 = require("../errors/response-errors/not-found.error");
const test_runner_service_ee_1 = require("../evaluation.ee/test-runner/test-runner.service.ee");
const telemetry_1 = require("../telemetry");
const workflow_history_service_1 = require("../workflows/workflow-history/workflow-history.service");
let EvaluationCollectionService = class EvaluationCollectionService {
    constructor(collectionRepo, testRunRepo, evalConfigRepo, workflowHistoryRepo, publishedVersionRepo, workflowHistoryService, testRunnerService, telemetry) {
        this.collectionRepo = collectionRepo;
        this.testRunRepo = testRunRepo;
        this.evalConfigRepo = evalConfigRepo;
        this.workflowHistoryRepo = workflowHistoryRepo;
        this.publishedVersionRepo = publishedVersionRepo;
        this.workflowHistoryService = workflowHistoryService;
        this.testRunnerService = testRunnerService;
        this.telemetry = telemetry;
    }
    async createCollection(user, workflowId, input) {
        const config = await this.evalConfigRepo.findByIdAndWorkflowId(input.evaluationConfigId, workflowId);
        if (!config) {
            throw new not_found_error_1.NotFoundError('EvaluationConfig not found for this workflow');
        }
        for (const [index, v] of input.versions.entries()) {
            if (v.workflowVersionId) {
                const exists = await this.workflowHistoryService.findVersion(workflowId, v.workflowVersionId);
                if (!exists) {
                    throw new bad_request_error_1.BadRequestError(`versions[${index}]: workflow version "${v.workflowVersionId}" not found`);
                }
            }
            if (v.existingTestRunId) {
                const run = await this.testRunRepo.findOneBy({ id: v.existingTestRunId });
                if (!run ||
                    run.workflowId !== workflowId ||
                    run.evaluationConfigId !== input.evaluationConfigId) {
                    throw new bad_request_error_1.BadRequestError(`versions[${index}]: test run "${v.existingTestRunId}" is not compatible with this collection`);
                }
                if (v.workflowVersionId && run.workflowVersionId !== v.workflowVersionId) {
                    throw new bad_request_error_1.BadRequestError(`versions[${index}]: test run "${v.existingTestRunId}" was executed against version "${run.workflowVersionId ?? '(unpinned)'}", not the requested "${v.workflowVersionId}"`);
                }
                if (!run.workflowVersionId) {
                    throw new bad_request_error_1.BadRequestError(`versions[${index}]: test run "${v.existingTestRunId}" has no pinned workflow version and cannot be reused in a collection`);
                }
            }
        }
        const collection = await this.collectionRepo.createCollection({
            id: (0, nanoid_1.nanoid)(),
            name: input.name,
            description: input.description ?? null,
            workflowId,
            evaluationConfigId: input.evaluationConfigId,
            createdById: user.id,
        });
        const existingRunIds = input.versions
            .filter((v) => v.existingTestRunId)
            .map((v) => v.existingTestRunId);
        if (existingRunIds.length > 0) {
            await this.collectionRepo.addRunsToCollection(collection.id, existingRunIds);
        }
        const configSnapshot = {
            ...config,
            createdAt: config.createdAt.toISOString(),
            updatedAt: config.updatedAt.toISOString(),
        };
        const runsStartedIds = [];
        for (const v of input.versions) {
            if (v.existingTestRunId)
                continue;
            const versionId = v.workflowVersionId ??
                (await this.workflowHistoryService.snapshotCurrent(workflowId)).versionId;
            const { testRun } = await this.testRunnerService.startTestRun(user, workflowId, input.concurrency ?? 1, {
                collectionId: collection.id,
                workflowVersionId: versionId,
                evaluationConfigId: input.evaluationConfigId,
                evaluationConfigSnapshot: configSnapshot,
            });
            runsStartedIds.push(testRun.id);
        }
        this.telemetry.track('Eval collection created', {
            user_id: user.id,
            workflow_id: workflowId,
            collection_id: collection.id,
            evaluation_config_id: input.evaluationConfigId,
            version_count: input.versions.length,
            existing_run_count: existingRunIds.length,
            new_run_count: runsStartedIds.length,
            dataset_id: this.extractDatasetId(config),
        });
        const record = this.toRecord(collection, existingRunIds.length + runsStartedIds.length);
        return { record, runsStartedIds };
    }
    async listCollections(workflowId) {
        const items = await this.collectionRepo.listByWorkflowId(workflowId);
        return items.map(({ runCount, ...c }) => this.toRecord(c, runCount));
    }
    async getCollectionDetail(workflowId, collectionId) {
        const detail = await this.collectionRepo.getDetailByIdAndWorkflowId(collectionId, workflowId);
        if (!detail)
            throw new not_found_error_1.NotFoundError('Collection not found');
        const runs = detail.runs.map((r) => this.toRunSummary(r));
        return { ...this.toRecord(detail.collection, runs.length), runs };
    }
    async updateCollectionMeta(workflowId, collectionId, input) {
        const updated = await this.collectionRepo.updateMeta(collectionId, workflowId, input);
        if (!updated)
            throw new not_found_error_1.NotFoundError('Collection not found');
        const runCount = await this.testRunRepo.count({ where: { collectionId } });
        return this.toRecord(updated, runCount);
    }
    async deleteCollection(user, workflowId, collectionId) {
        const owned = await this.collectionRepo.findByIdAndWorkflowId(collectionId, workflowId);
        if (!owned)
            throw new not_found_error_1.NotFoundError('Collection not found');
        const active = await this.testRunRepo.find({
            where: [
                { collectionId, status: 'running' },
                { collectionId, status: 'new' },
            ],
            select: ['id'],
        });
        if (active.length > 0) {
            await this.testRunnerService.cancelCollection(collectionId);
        }
        const { deleted, runsUnlinked } = await this.collectionRepo.deleteByIdAndWorkflowId(collectionId, workflowId);
        if (!deleted)
            throw new not_found_error_1.NotFoundError('Collection not found');
        this.telemetry.track('Eval collection deleted', {
            user_id: user.id,
            workflow_id: workflowId,
            collection_id: collectionId,
            runs_unlinked: runsUnlinked,
        });
        return { runsUnlinked };
    }
    async addRunToCollection(workflowId, collectionId, testRunId) {
        const collection = await this.collectionRepo.findByIdAndWorkflowId(collectionId, workflowId);
        if (!collection)
            throw new not_found_error_1.NotFoundError('Collection not found');
        const run = await this.testRunRepo.findOneBy({ id: testRunId });
        if (!run || run.workflowId !== workflowId) {
            throw new not_found_error_1.NotFoundError('Test run not found for this workflow');
        }
        if (run.evaluationConfigId !== collection.evaluationConfigId) {
            throw new bad_request_error_1.BadRequestError('Test run is not compatible with this collection (different evaluation config)');
        }
        if (!run.workflowVersionId) {
            throw new bad_request_error_1.BadRequestError('Test run has no pinned workflow version and cannot be added to a collection');
        }
        await this.collectionRepo.addRunsToCollection(collectionId, [testRunId]);
        await this.collectionRepo.updateInsightsCache(collectionId, null);
        return await this.getCollectionDetail(workflowId, collectionId);
    }
    async removeRunFromCollection(workflowId, collectionId, testRunId) {
        const collection = await this.collectionRepo.findByIdAndWorkflowId(collectionId, workflowId);
        if (!collection)
            throw new not_found_error_1.NotFoundError('Collection not found');
        const affected = await this.collectionRepo.removeRunFromCollection(collectionId, testRunId);
        if (affected === 0) {
            throw new not_found_error_1.NotFoundError('Test run is not part of this collection');
        }
        await this.collectionRepo.updateInsightsCache(collectionId, null);
        return await this.getCollectionDetail(workflowId, collectionId);
    }
    async getEvalVersions(workflowId, evaluationConfigId) {
        const config = await this.evalConfigRepo.findByIdAndWorkflowId(evaluationConfigId, workflowId);
        if (!config) {
            throw new not_found_error_1.NotFoundError('EvaluationConfig not found for this workflow');
        }
        const history = await this.workflowHistoryRepo.find({
            where: { workflowId },
            order: { createdAt: 'DESC' },
            select: ['versionId', 'name', 'autosaved', 'createdAt'],
        });
        const lastRuns = history.length === 0
            ? []
            : await this.testRunRepo.find({
                where: {
                    evaluationConfigId,
                    workflowVersionId: (0, typeorm_1.In)(history.map((h) => h.versionId)),
                },
                order: { createdAt: 'DESC' },
            });
        const latestRunByVersion = new Map();
        for (const run of lastRuns) {
            if (run.workflowVersionId && !latestRunByVersion.has(run.workflowVersionId)) {
                latestRunByVersion.set(run.workflowVersionId, run);
            }
        }
        const publishedRow = await this.publishedVersionRepo.findOneBy({ workflowId });
        const publishedVersionId = publishedRow?.publishedVersionId ?? null;
        const versions = [];
        versions.push({
            workflowVersionId: null,
            label: 'Current draft',
            sourceLabel: 'Live workflow',
            isCurrent: true,
            lastRun: null,
        });
        for (const h of history) {
            const lastRun = latestRunByVersion.get(h.versionId) ?? null;
            versions.push({
                workflowVersionId: h.versionId,
                label: h.name ?? this.shortVersionLabel(h.versionId),
                sourceLabel: this.formatSourceLabel(h, h.versionId === publishedVersionId),
                isCurrent: false,
                lastRun: lastRun
                    ? {
                        testRunId: lastRun.id,
                        runAt: (lastRun.runAt ?? lastRun.createdAt).toISOString(),
                        status: lastRun.status,
                        avgScore: this.computeAvgScore(lastRun),
                        isBest: false,
                        isCritical: false,
                    }
                    : null,
            });
        }
        const scored = versions.filter((v) => v.lastRun?.avgScore !== null && v.lastRun !== null);
        if (scored.length > 0) {
            const best = scored.reduce((acc, v) => (v.lastRun.avgScore ?? -Infinity) > (acc.lastRun.avgScore ?? -Infinity) ? v : acc);
            if (best.lastRun)
                best.lastRun.isBest = true;
            for (const v of scored) {
                if (v.lastRun && v.lastRun.avgScore !== null && v.lastRun.avgScore < 0.6) {
                    v.lastRun.isCritical = true;
                }
            }
        }
        return { evaluationConfigId, versions };
    }
    toRecord(collection, runCount) {
        return {
            id: collection.id,
            name: collection.name,
            description: collection.description,
            workflowId: collection.workflowId,
            evaluationConfigId: collection.evaluationConfigId,
            createdById: collection.createdById,
            createdAt: collection.createdAt.toISOString(),
            updatedAt: collection.updatedAt.toISOString(),
            runCount,
        };
    }
    toRunSummary(run) {
        return {
            testRunId: run.id,
            workflowVersionId: run.workflowVersionId,
            status: run.status,
            runAt: run.runAt ? run.runAt.toISOString() : null,
            completedAt: run.completedAt ? run.completedAt.toISOString() : null,
            avgScore: this.computeAvgScore(run),
            metrics: this.coerceMetrics(run.metrics),
        };
    }
    computeAvgScore(run) {
        const coerced = this.coerceMetrics(run.metrics);
        if (!coerced)
            return null;
        const values = Object.values(coerced);
        if (values.length === 0)
            return null;
        const sum = values.reduce((acc, v) => acc + v, 0);
        return sum / values.length;
    }
    coerceMetrics(metrics) {
        if (!metrics)
            return null;
        const out = {};
        for (const [k, v] of Object.entries(metrics)) {
            if (typeof v === 'number')
                out[k] = v;
            else if (typeof v === 'boolean')
                out[k] = v ? 1 : 0;
        }
        return Object.keys(out).length > 0 ? out : null;
    }
    extractDatasetId(config) {
        if (config.datasetSource === 'data_table' &&
            typeof config.datasetRef === 'object' &&
            config.datasetRef !== null &&
            'dataTableId' in config.datasetRef) {
            return String(config.datasetRef.dataTableId);
        }
        return null;
    }
    shortVersionLabel(versionId) {
        return `v ${versionId.slice(0, 8)}`;
    }
    formatDateLabel(date) {
        return date.toISOString().slice(0, 10);
    }
    formatSourceLabel(h, isPublished) {
        const date = this.formatDateLabel(h.createdAt);
        if (isPublished)
            return `Published · ${date}`;
        if (h.name)
            return `Named · ${date}`;
        if (h.autosaved)
            return `Autosaved · ${date}`;
        return `Snapshot · ${date}`;
    }
};
exports.EvaluationCollectionService = EvaluationCollectionService;
exports.EvaluationCollectionService = EvaluationCollectionService = __decorate([
    (0, di_1.Service)(),
    __metadata("design:paramtypes", [db_1.EvaluationCollectionRepository,
        db_1.TestRunRepository,
        db_1.EvaluationConfigRepository,
        db_1.WorkflowHistoryRepository,
        db_1.WorkflowPublishedVersionRepository,
        workflow_history_service_1.WorkflowHistoryService,
        test_runner_service_ee_1.TestRunnerService,
        telemetry_1.Telemetry])
], EvaluationCollectionService);
//# sourceMappingURL=evaluation-collection.service.js.map