n8n-nodes-base
Version:
Base nodes of n8n
255 lines • 13.2 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.metricHandlers = void 0;
const prompts_1 = require("@langchain/core/prompts");
const fastest_levenshtein_1 = require("fastest-levenshtein");
const n8n_workflow_1 = require("n8n-workflow");
const zod_1 = require("zod");
const utils_1 = require("../../Set/v2/helpers/utils");
const CannedMetricPrompts_ee_1 = require("../Evaluation/CannedMetricPrompts.ee");
exports.metricHandlers = {
async customMetrics(i) {
const dataToSave = this.getNodeParameter('metrics', i, {});
return Object.fromEntries((dataToSave?.assignments ?? []).map((assignment) => {
const assignmentValue = typeof assignment.value === 'number' ? assignment.value : Number(assignment.value);
if (isNaN(assignmentValue)) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Value for '${assignment.name}' isn't a number`, {
description: `It's currently '${assignment.value}'. Metrics must be numeric.`,
});
}
if (!assignment.name || isNaN(assignmentValue)) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Metric name missing', {
description: 'Make sure each metric you define has a name',
});
}
const { name, value } = (0, utils_1.validateEntry)(assignment.name, assignment.type, assignmentValue, this.getNode(), i, false, 1);
return [name, value];
}));
},
async toolsUsed(i) {
const expectedToolsParam = this.getNodeParameter('expectedTools', i, '');
const expectedToolsString = expectedToolsParam?.trim() || '';
const expectedTools = expectedToolsString
? expectedToolsString
.split(',')
.map((tool) => tool.trim())
.filter((tool) => tool !== '')
: [];
const intermediateSteps = this.getNodeParameter('intermediateSteps', i, {});
if (!expectedTools || expectedTools.length === 0) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Expected tool name missing', {
description: 'Make sure you add at least one expected tool name (comma-separated if multiple)',
});
}
if (!intermediateSteps || !Array.isArray(intermediateSteps)) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Intermediate steps missing', {
description: "Make sure to enable returning intermediate steps in your agent node's options, then map them in here",
});
}
// Convert user-entered tool names to the format used in intermediate steps (case-insensitive)
const normalizedExpectedTools = expectedTools.map((tool) => (0, n8n_workflow_1.nodeNameToToolName)(tool).toLowerCase());
// Calculate individual tool usage (1 if used, 0 if not used)
const toolUsageScores = normalizedExpectedTools.map((normalizedTool) => {
return intermediateSteps.some((step) => {
// Handle malformed intermediate steps gracefully
if (!step || !step.action || typeof step.action.tool !== 'string') {
return false;
}
return step.action.tool.toLowerCase() === normalizedTool;
})
? 1
: 0;
});
// Calculate the average of all tool usage scores
const averageScore = toolUsageScores.reduce((sum, score) => sum + score, 0) /
toolUsageScores.length;
const metricName = this.getNodeParameter('options.metricName', i, 'Tools Used');
return {
[metricName]: averageScore,
};
},
async categorization(i) {
const expectedAnswer = this.getNodeParameter('expectedAnswer', i, '')
.toString()
.trim();
const actualAnswer = this.getNodeParameter('actualAnswer', i, '').toString().trim();
if (!expectedAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Expected answer is missing', {
description: 'Make sure to fill in an expected answer',
});
}
if (!actualAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Actual answer is missing', {
description: 'Make sure to fill in an actual answer',
});
}
const metricName = this.getNodeParameter('options.metricName', i, 'Categorization');
return {
[metricName]: expectedAnswer === actualAnswer ? 1 : 0,
};
},
async stringSimilarity(i) {
const expectedAnswer = this.getNodeParameter('expectedAnswer', i, '')
.toString()
.trim();
const actualAnswer = this.getNodeParameter('actualAnswer', i, '').toString().trim();
if (!expectedAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Expected answer is missing', {
description: 'Make sure to fill in an expected answer',
});
}
if (!actualAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Actual answer is missing', {
description: 'Make sure to fill in an actual answer',
});
}
const metricName = this.getNodeParameter('options.metricName', i, 'String similarity');
const editDistance = (0, fastest_levenshtein_1.distance)(expectedAnswer, actualAnswer);
const longerStringLength = Math.max(expectedAnswer.length, actualAnswer.length);
const similarity = longerStringLength === 0 ? 1 : 1 - editDistance / longerStringLength;
return {
[metricName]: similarity,
};
},
async helpfulness(i) {
const userQuery = this.getNodeParameter('userQuery', i, '').toString().trim();
const actualAnswer = this.getNodeParameter('actualAnswer', i, '').toString().trim();
if (!userQuery) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'User query is missing', {
description: 'Make sure to fill in the user query in the User Query field',
});
}
if (!actualAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Response is missing', {
description: 'Make sure to fill in the response to evaluate in the Response field',
});
}
// Get the connected LLM model
const llm = (await this.getInputConnectionData('ai_languageModel', 0));
if (!llm) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No language model connected', {
description: 'Connect a language model to the Model input to use the helpfulness metric',
});
}
// Get the system prompt and input prompt template, using defaults if not provided
const systemPrompt = this.getNodeParameter('prompt', i, CannedMetricPrompts_ee_1.HELPFULNESS_PROMPT);
const inputPromptTemplate = this.getNodeParameter('options.inputPrompt', i, CannedMetricPrompts_ee_1.HELPFULNESS_INPUT_PROMPT[0]);
// Define the expected response schema
const responseSchema = zod_1.z.object({
extended_reasoning: zod_1.z
.string()
.describe('detailed step-by-step analysis of the response helpfulness'),
reasoning_summary: zod_1.z.string().describe('one sentence summary of the response helpfulness'),
score: zod_1.z
.number()
.int()
.min(1)
.max(5)
.describe('integer from 1 to 5 representing the helpfulness score'),
});
// Create LangChain prompt templates
const systemMessageTemplate = prompts_1.SystemMessagePromptTemplate.fromTemplate('{systemPrompt}');
const humanMessageTemplate = prompts_1.HumanMessagePromptTemplate.fromTemplate(inputPromptTemplate);
// Create the chat prompt template
const chatPrompt = prompts_1.ChatPromptTemplate.fromMessages([
systemMessageTemplate,
humanMessageTemplate,
]);
// Create chain with structured output
if (!llm.withStructuredOutput) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Language model does not support structured output', {
description: 'The connected language model does not support structured output. Please use a compatible model.',
});
}
const chain = chatPrompt.pipe(llm.withStructuredOutput(responseSchema));
try {
const response = await chain.invoke({
systemPrompt,
user_query: userQuery,
actual_answer: actualAnswer,
});
const metricName = this.getNodeParameter('options.metricName', i, 'Helpfulness');
// Return the score as the main metric
return {
[metricName]: response.score,
};
}
catch (error) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Failed to evaluate helpfulness', {
description: `Error from language model: ${error instanceof Error ? error.message : String(error)}`,
});
}
},
async correctness(i) {
const expectedAnswer = this.getNodeParameter('expectedAnswer', i, '')
.toString()
.trim();
const actualAnswer = this.getNodeParameter('actualAnswer', i, '').toString().trim();
if (!expectedAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Expected answer is missing', {
description: 'Make sure to fill in an expected answer',
});
}
if (!actualAnswer) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Actual answer is missing', {
description: 'Make sure to fill in an actual answer',
});
}
// Get the connected LLM model
const llm = (await this.getInputConnectionData('ai_languageModel', 0));
if (!llm) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No language model connected', {
description: 'Connect a language model to the Model input to use the correctness metric',
});
}
// Get the system prompt and input prompt template, using defaults if not provided
const systemPrompt = this.getNodeParameter('prompt', i, CannedMetricPrompts_ee_1.CORRECTNESS_PROMPT);
const inputPromptTemplate = this.getNodeParameter('options.inputPrompt', i, CannedMetricPrompts_ee_1.CORRECTNESS_INPUT_PROMPT[0]);
// Define the expected response schema
const responseSchema = zod_1.z.object({
extended_reasoning: zod_1.z
.string()
.describe('detailed step-by-step analysis of factual accuracy and similarity'),
reasoning_summary: zod_1.z.string().describe('one sentence summary focusing on key differences'),
score: zod_1.z
.number()
.int()
.min(1)
.max(5)
.describe('integer from 1 to 5 representing the similarity score'),
});
// Create LangChain prompt templates
const systemMessageTemplate = prompts_1.SystemMessagePromptTemplate.fromTemplate('{systemPrompt}');
const humanMessageTemplate = prompts_1.HumanMessagePromptTemplate.fromTemplate(inputPromptTemplate);
// Create the chat prompt template
const chatPrompt = prompts_1.ChatPromptTemplate.fromMessages([
systemMessageTemplate,
humanMessageTemplate,
]);
// Create chain with structured output
if (!llm.withStructuredOutput) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Language model does not support structured output', {
description: 'The connected language model does not support structured output. Please use a compatible model.',
});
}
const chain = chatPrompt.pipe(llm.withStructuredOutput(responseSchema));
try {
const response = await chain.invoke({
systemPrompt,
actual_answer: actualAnswer,
expected_answer: expectedAnswer,
});
const metricName = this.getNodeParameter('options.metricName', i, 'Correctness');
// Return the score as the main metric
return {
[metricName]: response.score,
};
}
catch (error) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Failed to evaluate correctness', {
description: `Error from language model: ${error instanceof Error ? error.message : String(error)}`,
});
}
},
};
//# sourceMappingURL=metricHandlers.js.map