@juspay/neurolink
Version:
Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio
471 lines • 15.7 kB
JavaScript
/**
* @file Format Scorer
* Evaluates response format compliance (JSON, markdown, code, etc.)
*/
import { logger } from "../../../utils/logger.js";
import { BaseRuleScorer, DEFAULT_RULE_SCORER_CONFIG, } from "./baseRuleScorer.js";
/**
* Scorer metadata for format
*/
const FORMAT_METADATA = {
id: "format",
name: "Format Validator",
description: "Evaluates response format compliance (JSON, markdown, code, lists, etc.)",
type: "rule",
category: "quality",
version: "1.0.0",
defaultConfig: {
...DEFAULT_RULE_SCORER_CONFIG,
threshold: 0.8,
},
requiredInputs: ["response"],
optionalInputs: ["custom"],
};
/**
* FormatScorer evaluates response format against expected formats
*/
export class FormatScorer extends BaseRuleScorer {
_formatConfig;
constructor(config) {
super(FORMAT_METADATA, config);
this._formatConfig = {
expectedFormat: "plain",
strictFormat: false,
...config,
};
}
/**
* Get format-specific configuration
*/
get formatConfig() {
return this._formatConfig;
}
/**
* Get rules for this scorer
*/
getRules() {
const rules = [];
const formats = this._formatConfig.allowedFormats ?? [
this._formatConfig.expectedFormat ?? "plain",
];
// Main format rule
rules.push({
id: "format-check",
description: `Check format is one of: ${formats.join(", ")}`,
type: "custom",
params: {
formats,
strict: this._formatConfig.strictFormat ?? false,
},
weight: 1.0,
});
// Add specific requirement rules based on format
if (formats.includes("markdown") &&
this._formatConfig.markdownRequirements) {
rules.push({
id: "markdown-requirements",
description: "Check markdown structure requirements",
type: "custom",
params: {
requirements: this._formatConfig.markdownRequirements,
},
weight: 0.5,
});
}
if ((formats.includes("list") ||
formats.includes("numbered-list") ||
formats.includes("bullet-list")) &&
this._formatConfig.listRequirements) {
rules.push({
id: "list-requirements",
description: "Check list structure requirements",
type: "custom",
params: {
requirements: this._formatConfig.listRequirements,
},
weight: 0.5,
});
}
if (formats.includes("json") && this._formatConfig.jsonSchema) {
rules.push({
id: "json-schema",
description: "Validate JSON against schema",
type: "custom",
params: {
schema: this._formatConfig
.jsonSchema,
},
weight: 0.5,
});
}
return rules;
}
/**
* Evaluate a single format rule
*/
evaluateRule(rule, input) {
switch (rule.id) {
case "format-check": {
const formats = rule.params.formats;
const result = this._validateFormat(input.response, formats);
return {
passed: result.isValid,
score: result.isValid ? 1.0 : 0.0,
};
}
case "markdown-requirements": {
const requirements = rule.params
.requirements;
const result = this._checkMarkdownRequirements(input.response, requirements ?? {});
return result;
}
case "list-requirements": {
const requirements = rule.params
.requirements;
const result = this._checkListRequirements(input.response, requirements ?? {});
return result;
}
case "json-schema": {
const schema = rule.params.schema;
const result = this._validateJsonSchema(input.response, schema);
return result;
}
default:
return { passed: true, score: 1.0 };
}
}
/**
* Validate format against allowed formats
*/
_validateFormat(text, allowedFormats) {
const issues = [];
// Detect format
const detectedFormat = this._detectFormat(text);
// Check if detected format is allowed
const isValid = allowedFormats.includes(detectedFormat) ||
(detectedFormat === "plain" && allowedFormats.includes("plain"));
if (!isValid) {
issues.push(`Expected format(s): ${allowedFormats.join(", ")}, but detected: ${detectedFormat}`);
}
return { isValid, detectedFormat, issues };
}
/**
* Detect the format of the text
*/
_detectFormat(text) {
const trimmed = text.trim();
// Check JSON
if (this._isValidJson(trimmed)) {
return "json";
}
// Check YAML (basic detection)
if (this._isYaml(trimmed)) {
return "yaml";
}
// Check XML/HTML
if (this._isXml(trimmed)) {
return trimmed.toLowerCase().includes("<!doctype html") ||
trimmed.includes("<html")
? "html"
: "xml";
}
// Check code blocks
if (this._hasCodeBlocks(trimmed)) {
return "code";
}
// Check markdown elements
if (this._hasMarkdownElements(trimmed)) {
return "markdown";
}
// Check lists
if (this._isNumberedList(trimmed)) {
return "numbered-list";
}
if (this._isBulletList(trimmed)) {
return "bullet-list";
}
// Check tables (markdown style)
if (this._hasTable(trimmed)) {
return "table";
}
return "plain";
}
/**
* Check if text is valid JSON
*/
_isValidJson(text) {
try {
const parsed = JSON.parse(text);
return typeof parsed === "object" && parsed !== null;
}
catch {
return false;
}
}
/**
* Check if text appears to be YAML
*/
_isYaml(text) {
// Basic YAML detection: key: value patterns
const lines = text.split("\n");
let yamlPatternCount = 0;
for (const line of lines) {
// Skip empty lines and comments
if (line.trim() === "" || line.trim().startsWith("#")) {
continue;
}
// Check for key: value pattern (but not URLs)
if (/^[\s-]*[\w_]+:\s*.+/.test(line) && !line.includes("://")) {
yamlPatternCount++;
}
}
return yamlPatternCount >= 2;
}
/**
* Check if text is XML
*/
_isXml(text) {
return ((text.startsWith("<?xml") || text.startsWith("<")) &&
text.endsWith(">") &&
/<\/?[a-zA-Z][a-zA-Z0-9]*[^>]{0,1000}>/.test(text.slice(0, 10000)));
}
/**
* Check if text has code blocks
*/
_hasCodeBlocks(text) {
return /```[\s\S]{0,10000}?```/.test(text) || /`[^`]{1,1000}`/.test(text);
}
/**
* Check if text has markdown elements
*/
_hasMarkdownElements(text) {
// Check for headings, bold, italic, links, etc.
const markdownPatterns = [
/^#{1,6}\s+.+/m, // Headings
/\*\*[^*]+\*\*/, // Bold
/\*[^*]+\*/, // Italic
/__[^_]+__/, // Bold (underscore)
/_[^_]+_/, // Italic (underscore)
/\[[^\]]{1,500}\]\([^)]{1,2000}\)/, // Links (bounded)
/!\[[^\]]{0,500}\]\([^)]{1,2000}\)/, // Images (bounded)
/^>\s+.+/m, // Blockquotes
/^-{3,}$/m, // Horizontal rule
/^\*{3,}$/m, // Horizontal rule
];
return markdownPatterns.some((pattern) => pattern.test(text));
}
/**
* Check if text is a numbered list
*/
_isNumberedList(text) {
const lines = text
.split("\n")
.map((l) => l.trim())
.filter((l) => l.length > 0);
const numberedLines = lines.filter((l) => /^\d+[.)]\s+/.test(l));
return (numberedLines.length >= 2 && numberedLines.length / lines.length > 0.5);
}
/**
* Check if text is a bullet list
*/
_isBulletList(text) {
const lines = text
.split("\n")
.map((l) => l.trim())
.filter((l) => l.length > 0);
const bulletLines = lines.filter((l) => /^[-*+]\s+/.test(l));
return bulletLines.length >= 2 && bulletLines.length / lines.length > 0.5;
}
/**
* Check if text has a table
*/
_hasTable(text) {
// Markdown table pattern: | header | header |
return /\|.+\|/.test(text) && /\|[-:]+\|/.test(text);
}
/**
* Check markdown-specific requirements
*/
_checkMarkdownRequirements(text, requirements) {
let totalChecks = 0;
let passedChecks = 0;
if (requirements.hasHeadings !== undefined) {
totalChecks++;
if (/^#{1,6}\s+.+/m.test(text) === requirements.hasHeadings) {
passedChecks++;
}
}
if (requirements.hasCodeBlocks !== undefined) {
totalChecks++;
if (this._hasCodeBlocks(text) === requirements.hasCodeBlocks) {
passedChecks++;
}
}
if (requirements.hasLinks !== undefined) {
totalChecks++;
if (/\[[^\]]{1,500}\]\([^)]{1,2000}\)/.test(text) === requirements.hasLinks) {
passedChecks++;
}
}
if (requirements.hasLists !== undefined) {
totalChecks++;
const hasList = this._isNumberedList(text) || this._isBulletList(text);
if (hasList === requirements.hasLists) {
passedChecks++;
}
}
if (requirements.minHeadingLevel !== undefined ||
requirements.maxHeadingLevel !== undefined) {
totalChecks++;
const headingMatches = text.match(/^(#{1,6})\s+/gm);
if (headingMatches) {
const levels = headingMatches.map((m) => m.trim().indexOf(" "));
const minLevel = Math.min(...levels);
const maxLevel = Math.max(...levels);
const minOk = requirements.minHeadingLevel === undefined ||
minLevel >= requirements.minHeadingLevel;
const maxOk = requirements.maxHeadingLevel === undefined ||
maxLevel <= requirements.maxHeadingLevel;
if (minOk && maxOk) {
passedChecks++;
}
}
}
if (totalChecks === 0) {
return { passed: true, score: 1.0 };
}
const score = passedChecks / totalChecks;
return { passed: score >= 0.8, score };
}
/**
* Check list-specific requirements
*/
_checkListRequirements(text, requirements) {
const lines = text
.split("\n")
.map((l) => l.trim())
.filter((l) => l.length > 0);
const listItems = lines.filter((l) => /^(\d+[.)]\s+|[-*+]\s+)/.test(l));
let totalChecks = 0;
let passedChecks = 0;
if (requirements.minItems !== undefined) {
totalChecks++;
if (listItems.length >= requirements.minItems) {
passedChecks++;
}
}
if (requirements.maxItems !== undefined) {
totalChecks++;
if (listItems.length <= requirements.maxItems) {
passedChecks++;
}
}
if (requirements.itemPattern !== undefined) {
totalChecks++;
if (requirements.itemPattern.length > 100) {
logger.warn("[FormatScorer] itemPattern too long, using default");
}
else if (/(\+|\*|\?)\s*\).*?(\+|\*|\?)/.test(requirements.itemPattern) ||
/\(\?.*?\)\s*(\+|\*|\{)/.test(requirements.itemPattern)) {
// Detect nested quantifiers that can cause catastrophic backtracking
logger.warn("[FormatScorer] itemPattern contains potentially unsafe nested quantifiers");
}
else {
try {
const pattern = new RegExp(requirements.itemPattern);
const matchingItems = listItems.filter((item) => pattern.test(item));
if (matchingItems.length === listItems.length) {
passedChecks++;
}
}
catch {
logger.warn("[FormatScorer] Invalid itemPattern, using default");
}
}
}
if (totalChecks === 0) {
return { passed: true, score: 1.0 };
}
const score = passedChecks / totalChecks;
return { passed: score >= 0.8, score };
}
/**
* Validate JSON against schema (basic validation)
*/
_validateJsonSchema(text, _schema) {
// First check if it's valid JSON
try {
JSON.parse(text);
// TODO: Implement full JSON Schema validation
// For now, just check it's valid JSON
return { passed: true, score: 1.0 };
}
catch {
return { passed: false, score: 0.0 };
}
}
/**
* Override score to add detailed format analysis
*/
async score(input) {
const result = await super.score(input);
const detectedFormat = this._detectFormat(input.response);
return {
...result,
metadata: {
...result.metadata,
detectedFormat,
expectedFormat: this._formatConfig.expectedFormat ?? "plain",
allowedFormats: this._formatConfig.allowedFormats ?? [],
strictFormat: this._formatConfig.strictFormat ?? false,
},
};
}
}
/**
* Factory function for creating FormatScorer instances
*/
export async function createFormatScorer(config) {
return new FormatScorer(config);
}
/**
* Pre-configured format scorer presets
*/
export const FormatScorerPresets = {
/** JSON format */
json: () => new FormatScorer({
expectedFormat: "json",
strictFormat: true,
}),
/** Markdown format */
markdown: () => new FormatScorer({
expectedFormat: "markdown",
}),
/** Markdown with headings required */
markdownWithHeadings: () => new FormatScorer({
expectedFormat: "markdown",
markdownRequirements: {
hasHeadings: true,
minHeadingLevel: 1,
maxHeadingLevel: 3,
},
}),
/** Bullet list format */
bulletList: () => new FormatScorer({
expectedFormat: "bullet-list",
}),
/** Numbered list format */
numberedList: () => new FormatScorer({
expectedFormat: "numbered-list",
}),
/** Code response */
code: () => new FormatScorer({
expectedFormat: "code",
}),
/** Plain text only */
plainText: () => new FormatScorer({
expectedFormat: "plain",
strictFormat: true,
}),
};
//# sourceMappingURL=formatScorer.js.map