@juspay/neurolink
Version:
Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio
421 lines • 13 kB
JavaScript
/**
* @file Scorer Builder
* Fluent builder API for creating custom scorers
*/
import { composeScorers, createFunctionScorer } from "./customScorerUtils.js";
/**
* Fluent builder for creating custom scorers
*/
export class ScorerBuilder {
_id;
_name;
_description;
_type = "rule";
_category = "custom";
_version = "1.0.0";
_requiredInputs = ["response"];
_optionalInputs = [
"query",
"context",
"groundTruth",
];
_threshold = 0.7;
_weight = 1.0;
_timeout = 5000;
_retries = 0;
_scorerFn;
_rules = [];
_subScorers = [];
_aggregation = "average";
_subScorerWeights = [];
constructor(id, name) {
this._id = id;
this._name = name;
}
/**
* Create a new scorer builder
*/
static create(id, name) {
return new ScorerBuilder(id, name);
}
/**
* Set scorer description
*/
description(desc) {
this._description = desc;
return this;
}
/**
* Set scorer type
*/
type(type) {
this._type = type;
return this;
}
/**
* Set scorer category
*/
category(category) {
this._category = category;
return this;
}
/**
* Set scorer version
*/
version(version) {
this._version = version;
return this;
}
/**
* Set required inputs
*/
requireInputs(...inputs) {
this._requiredInputs = inputs;
return this;
}
/**
* Set optional inputs
*/
optionalInputs(...inputs) {
this._optionalInputs = inputs;
return this;
}
/**
* Set pass/fail threshold
*/
threshold(threshold) {
this._threshold = threshold;
return this;
}
/**
* Set weight for aggregation
*/
weight(weight) {
this._weight = weight;
return this;
}
/**
* Set execution timeout
*/
timeout(ms) {
this._timeout = ms;
return this;
}
/**
* Set retry count
*/
retries(count) {
this._retries = count;
return this;
}
/**
* Set the scoring function
*/
scoringFunction(fn) {
this._scorerFn = fn;
return this;
}
/**
* Add a sub-scorer for composition
*/
addScorer(scorer, weight) {
this._subScorers.push(scorer);
this._subScorerWeights.push(weight ?? 1.0);
return this;
}
/**
* Set aggregation method for composed scorers
*/
aggregateWith(method) {
this._aggregation = method;
return this;
}
/**
* Add a regex check rule
*/
matchesPattern(pattern, options) {
const patternStr = typeof pattern === "string" ? pattern : pattern.source;
const flags = typeof pattern === "string" ? "gi" : pattern.flags;
// Validate pattern safety (same guards as regex scorer utilities)
if (patternStr.length > 200) {
throw new Error("Regex pattern exceeds maximum length of 200 characters");
}
if (/(\+|\*|\{)\S*(\+|\*|\{)/.test(patternStr)) {
throw new Error("Regex pattern contains nested quantifiers which may cause catastrophic backtracking");
}
// Pre-compile to validate the pattern at build time
try {
new RegExp(patternStr, flags);
}
catch (e) {
throw new Error(`Invalid regex pattern: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
}
this._rules.push({
id: options?.id ?? `regex-${this._rules.length}`,
description: `Match pattern: ${patternStr}`,
type: "regex",
params: {
pattern: patternStr,
flags,
},
weight: options?.weight ?? 1.0,
});
return this;
}
/**
* Add a keyword check rule
*/
containsKeyword(keyword, options) {
this._rules.push({
id: options?.id ?? `keyword-${this._rules.length}`,
description: `Contains keyword: ${keyword}`,
type: "keyword",
params: {
keyword,
caseInsensitive: true,
},
weight: options?.weight ?? 1.0,
});
return this;
}
/**
* Add a length check rule
*/
hasLength(options) {
this._rules.push({
id: options.id ?? `length-${this._rules.length}`,
description: "Length check",
type: "length",
params: {
minWords: options.minWords ?? null,
maxWords: options.maxWords ?? null,
minChars: options.minChars ?? null,
maxChars: options.maxChars ?? null,
},
weight: options.weight ?? 1.0,
});
return this;
}
/**
* Add a custom rule
*/
customRule(rule) {
this._rules.push(rule);
return this;
}
/**
* Build the scorer
*/
build() {
// If we have sub-scorers, create a composed scorer
if (this._subScorers.length > 0) {
return composeScorers(this._id, this._name, this._subScorers, {
aggregation: this._aggregation,
weights: this._subScorerWeights,
description: this._description,
config: this._buildConfig(),
});
}
// If we have a scoring function, use it
if (this._scorerFn) {
return createFunctionScorer(this._id, this._name, this._scorerFn, {
type: this._type,
version: this._version,
requiredInputs: this._requiredInputs,
optionalInputs: this._optionalInputs,
description: this._description,
category: this._category,
config: this._buildConfig(),
});
}
// If we have rules, create a rule-based scorer
if (this._rules.length > 0) {
return this._buildRuleScorer();
}
// Default: return a pass-through scorer
return createFunctionScorer(this._id, this._name, async () => ({
score: 10,
reasoning: "No scoring logic defined - passing by default",
}), {
type: this._type,
version: this._version,
requiredInputs: this._requiredInputs,
optionalInputs: this._optionalInputs,
description: this._description,
category: this._category,
config: this._buildConfig(),
});
}
/**
* Build configuration object
*/
_buildConfig() {
return {
enabled: true,
threshold: this._threshold,
weight: this._weight,
timeout: this._timeout,
retries: this._retries,
};
}
/**
* Build a rule-based scorer from accumulated rules
*/
_buildRuleScorer() {
const rules = this._rules;
const config = this._buildConfig();
return createFunctionScorer(this._id, this._name, async (input) => {
const results = [];
for (const rule of rules) {
const result = this._evaluateRule(rule, input);
results.push({ rule, ...result });
}
// Calculate weighted score
let totalWeight = 0;
let weightedScore = 0;
for (const result of results) {
const weight = result.rule.weight ?? 1.0;
totalWeight += weight;
weightedScore += result.score * weight;
}
const finalScore = totalWeight > 0 ? (weightedScore / totalWeight) * 10 : 10;
const passedCount = results.filter((r) => r.passed).length;
return {
score: finalScore,
reasoning: `${passedCount}/${rules.length} rules passed`,
metadata: {
ruleResults: results.map((r) => ({
ruleId: r.rule.id,
passed: r.passed,
score: r.score,
})),
},
};
}, {
type: this._type,
version: this._version,
requiredInputs: this._requiredInputs,
optionalInputs: this._optionalInputs,
description: this._description,
category: this._category,
config,
});
}
/**
* Evaluate a single rule
*/
_evaluateRule(rule, input) {
switch (rule.type) {
case "regex": {
let regex;
try {
regex = new RegExp(rule.params.pattern, rule.params.flags ?? "i");
}
catch {
return { passed: false, score: 0.0 };
}
if (regex.global) {
regex.lastIndex = 0;
}
const matches = regex.test(input.response);
return { passed: matches, score: matches ? 1.0 : 0.0 };
}
case "keyword": {
const keyword = rule.params.keyword;
const caseInsensitive = rule.params.caseInsensitive;
const text = caseInsensitive
? input.response.toLowerCase()
: input.response;
const search = caseInsensitive ? keyword.toLowerCase() : keyword;
const found = text.includes(search);
return { passed: found, score: found ? 1.0 : 0.0 };
}
case "length": {
const wordCount = input.response
.trim()
.split(/\s+/)
.filter((w) => w.length > 0).length;
const charCount = input.response.length;
const minWords = rule.params.minWords;
const maxWords = rule.params.maxWords;
const minChars = rule.params.minChars;
const maxChars = rule.params.maxChars;
let passed = true;
if (minWords !== null && wordCount < minWords) {
passed = false;
}
if (maxWords !== null && wordCount > maxWords) {
passed = false;
}
if (minChars !== null && charCount < minChars) {
passed = false;
}
if (maxChars !== null && charCount > maxChars) {
passed = false;
}
return { passed, score: passed ? 1.0 : 0.0 };
}
case "custom": {
// Execute custom evaluator if provided
if (rule.params?.evaluate &&
typeof rule.params.evaluate === "function") {
try {
const customResult = rule.params.evaluate(input);
if (typeof customResult?.passed === "boolean" &&
typeof customResult?.score === "number") {
return customResult;
}
}
catch {
return { passed: false, score: 0.0 };
}
}
// No evaluator provided - pass by default with warning
return { passed: true, score: 1.0 };
}
default:
return { passed: true, score: 1.0 };
}
}
}
/**
* Quick builder factory functions
*/
export const Scorers = {
/**
* Create a new scorer builder
*/
create: (id, name) => ScorerBuilder.create(id, name),
/**
* Create a simple pass/fail scorer based on a condition
*/
passIf: (id, name, condition) => ScorerBuilder.create(id, name).scoringFunction(async (input) => ({
score: condition(input) ? 10 : 0,
reasoning: condition(input) ? "Condition passed" : "Condition failed",
})),
/**
* Create a scorer that checks for required content
*/
requiresContent: (id, name, keywords) => {
const builder = ScorerBuilder.create(id, name).category("quality");
for (const keyword of keywords) {
builder.containsKeyword(keyword);
}
return builder;
},
/**
* Create a scorer with length constraints
*/
withLength: (id, name, options) => ScorerBuilder.create(id, name).category("quality").hasLength(options),
/**
* Create a scorer that combines multiple scorers
*/
combine: (id, name, scorers) => {
const builder = ScorerBuilder.create(id, name).category("custom");
for (const scorer of scorers) {
builder.addScorer(scorer);
}
return builder;
},
};
//# sourceMappingURL=scorerBuilder.js.map