n8n
Version:
n8n Workflow Automation Tool
192 lines • 10.5 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AgentEvalCaseGenerationService = void 0;
const agent_config_1 = require("@n8n/ai-utilities/agent-config");
const api_types_1 = require("@n8n/api-types");
const backend_common_1 = require("@n8n/backend-common");
const db_1 = require("@n8n/db");
const di_1 = require("@n8n/di");
const n8n_workflow_1 = require("n8n-workflow");
const credentials_service_1 = require("../../credentials/credentials.service");
const forbidden_error_1 = require("../../errors/response-errors/forbidden.error");
const source_control_preferences_service_ee_1 = require("../../modules/source-control.ee/source-control-preferences.service.ee");
const agent_evals_flag_gate_1 = require("./agent-evals-flag-gate");
const agent_config_service_1 = require("../agents/agent-config.service");
const case_generation_prompt_1 = require("./case-generation/case-generation-prompt");
const dimensions_1 = require("./case-generation/dimensions");
const credential_field_mapping_1 = require("../agents/json-config/credential-field-mapping");
const model_config_1 = require("../agents/json-config/model-config");
const agent_credential_provider_1 = require("../agents/utils/agent-credential-provider");
const data_table_service_1 = require("../data-table/data-table.service");
const data_table_name_conflict_error_1 = require("../data-table/errors/data-table-name-conflict.error");
const DEFAULT_CASE_COUNT = 6;
const MAX_CASE_COUNT = 20;
const MAX_CASE_TEXT_CHARS = 2_000;
const GENERATE_TIMEOUT_MS = 60_000;
const INPUT_COLUMN = 'input';
const CRITERIA_COLUMN = 'criteria';
const MAX_NAME_ATTEMPTS = 20;
let AgentEvalCaseGenerationService = class AgentEvalCaseGenerationService {
constructor(logger, agentConfigService, credentialsService, dataTableService, datasetRepository, flagGate, sourceControlPreferencesService) {
this.logger = logger;
this.agentConfigService = agentConfigService;
this.credentialsService = credentialsService;
this.dataTableService = dataTableService;
this.datasetRepository = datasetRepository;
this.flagGate = flagGate;
this.sourceControlPreferencesService = sourceControlPreferencesService;
}
async generateDraftCases(user, projectId, agentId, options = {}) {
await this.flagGate.assertEnabled(user);
this.assertInstanceWriteAccess();
const config = await this.agentConfigService.getConfig(agentId, projectId);
const modelConfig = await this.resolveAgentModel(config, projectId, user);
const count = clampCount(options.count);
const capabilities = (0, case_generation_prompt_1.deriveCapabilities)(config);
const tuples = (0, dimensions_1.sampleDimensionTuples)(capabilities, count);
const summary = (0, case_generation_prompt_1.buildAgentSummary)(config);
const generated = await this.invokeModel(modelConfig, (0, case_generation_prompt_1.buildCaseGenerationUserPrompt)(summary, tuples), tuples.length);
const cases = boundCases(generated, tuples.length);
const trimmedName = options.datasetName?.trim();
const baseName = trimmedName && trimmedName.length > 0 ? trimmedName : defaultDatasetName(config.name);
const { datasetId, dataTableId } = await this.persistDataset(projectId, agentId, user.id, baseName, cases);
this.logger.debug('Generated draft eval cases', {
agentId,
datasetId,
caseCount: cases.length,
});
return { datasetId, dataTableId, cases };
}
assertInstanceWriteAccess() {
if (this.sourceControlPreferencesService.getPreferences().branchReadOnly) {
throw new forbidden_error_1.ForbiddenError('Cannot generate eval cases on a protected instance. This instance is in read-only mode.');
}
}
async resolveAgentModel(config, projectId, user) {
const { model, credential } = config;
if (!model || !credential || credential === api_types_1.MANAGED_CREDENTIAL_TOKEN) {
throw new n8n_workflow_1.UserError('This agent needs a configured model and API-key credential before draft cases can be generated.');
}
if (!(0, credential_field_mapping_1.isSupportedAgentProvider)((0, agent_config_1.getProviderPrefix)(model))) {
throw new n8n_workflow_1.UserError(`The agent's model provider is not supported for case generation ("${model}").`);
}
const credentialProvider = (0, agent_credential_provider_1.createAgentCredentialProvider)(this.credentialsService, projectId, user);
return await (0, model_config_1.resolveCredentialAwareModelConfig)(model, credential, credentialProvider);
}
async invokeModel(modelConfig, userPrompt, expectedCount) {
const { Agent } = await import('@n8n/agents');
const agent = new Agent('agent-eval-case-generation')
.model(modelConfig)
.instructions(case_generation_prompt_1.CASE_GENERATION_SYSTEM_PROMPT)
.structuredOutput(case_generation_prompt_1.generatedCasesSchema);
const attempt = async (prompt) => {
const result = await agent.generate(prompt, {
abortSignal: AbortSignal.timeout(GENERATE_TIMEOUT_MS),
});
const parsed = case_generation_prompt_1.generatedCasesSchema.safeParse(result.structuredOutput);
if (!parsed.success)
return null;
const cases = parsed.data.cases
.map((c) => ({ input: c.input.trim(), whatToCheck: c.whatToCheck.trim() }))
.filter((c) => c.input.length > 0 && c.whatToCheck.length > 0);
return cases.length >= expectedCount ? cases : null;
};
const first = await attempt(userPrompt);
if (first)
return first;
const retry = await attempt(`${userPrompt}\n\nReturn ONLY a JSON object with exactly ${expectedCount} cases matching the required schema — no extra keys, no prose.`);
if (retry)
return retry;
throw new n8n_workflow_1.OperationalError('Case generation returned fewer valid cases than requested after a retry');
}
async persistDataset(projectId, agentId, createdById, baseName, cases) {
const columns = [
{ name: INPUT_COLUMN, type: 'string' },
{ name: CRITERIA_COLUMN, type: 'string' },
];
let table;
let name = baseName;
for (let attempt = 0; table === undefined; attempt++) {
name = attempt === 0 ? baseName : suffixedName(baseName, attempt + 1);
try {
table = await this.dataTableService.createDataTable(projectId, { name, columns });
}
catch (error) {
if (error instanceof data_table_name_conflict_error_1.DataTableNameConflictError && attempt < MAX_NAME_ATTEMPTS - 1) {
continue;
}
throw error;
}
}
try {
const rows = cases.map((c) => ({
[INPUT_COLUMN]: c.input,
[CRITERIA_COLUMN]: c.whatToCheck,
}));
await this.dataTableService.insertRows(table.id, projectId, rows);
const dataset = await this.datasetRepository.createDataset({
name,
agentId,
datasetSource: 'data_table',
datasetRef: { dataTableId: table.id },
columnMapping: { input: INPUT_COLUMN, criteria: CRITERIA_COLUMN },
createdById,
});
return { datasetId: dataset.id, dataTableId: table.id };
}
catch (error) {
await this.rollBackDataTable(table.id, projectId);
throw error;
}
}
async rollBackDataTable(dataTableId, projectId) {
try {
await this.dataTableService.deleteDataTable(dataTableId, projectId);
}
catch (cleanupError) {
this.logger.error('Failed to clean up data table after case-generation failure', {
dataTableId,
error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
});
}
}
};
exports.AgentEvalCaseGenerationService = AgentEvalCaseGenerationService;
exports.AgentEvalCaseGenerationService = AgentEvalCaseGenerationService = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [backend_common_1.Logger, agent_config_service_1.AgentConfigService, credentials_service_1.CredentialsService, data_table_service_1.DataTableService, db_1.AgentEvalDatasetRepository, agent_evals_flag_gate_1.AgentEvalsFlagGate, source_control_preferences_service_ee_1.SourceControlPreferencesService])
], AgentEvalCaseGenerationService);
function boundCases(cases, limit) {
return cases.slice(0, limit).map((c) => ({
input: truncateText(c.input, MAX_CASE_TEXT_CHARS),
whatToCheck: truncateText(c.whatToCheck, MAX_CASE_TEXT_CHARS),
}));
}
function truncateText(text, max) {
return text.length > max ? text.slice(0, max) : text;
}
function clampCount(count) {
if (count === undefined || !Number.isFinite(count))
return DEFAULT_CASE_COUNT;
return Math.min(Math.max(Math.trunc(count), 1), MAX_CASE_COUNT);
}
function defaultDatasetName(agentName) {
return truncateName(`Draft cases for ${agentName}`);
}
function suffixedName(baseName, n) {
const suffix = ` (${n})`;
return `${truncateName(baseName, 128 - suffix.length)}${suffix}`;
}
function truncateName(name, max = 128) {
return truncateText(name, max);
}
//# sourceMappingURL=agent-eval-case-generation.service.js.map