judgeval
Version:
Judgment SDK for TypeScript/JavaScript
314 lines • 11.5 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
/**
* Rules system for JudgEval that enables alerts based on metric thresholds.
*/
import { v4 as uuidv4 } from 'uuid';
import { ScorerWrapper } from './scorers/base-scorer.js';
/**
* Status of an alert evaluation
*/
export var AlertStatus;
(function (AlertStatus) {
AlertStatus["TRIGGERED"] = "triggered";
AlertStatus["NOT_TRIGGERED"] = "not_triggered";
})(AlertStatus || (AlertStatus = {}));
/**
* A single metric condition
*/
export class Condition {
constructor(metric) {
this.metric = metric;
}
/**
* Get the name of the metric for lookups in scores dictionary
*/
get metricName() {
if (this.metric instanceof ScorerWrapper) {
return this.metric.scoreType || String(this.metric);
}
else if ('scoreType' in this.metric) {
return this.metric.scoreType;
}
else {
return String(this.metric);
}
}
/**
* Get the threshold from the metric
*/
get threshold() {
return 'threshold' in this.metric ? this.metric.threshold : 0.5;
}
/**
* Evaluate the condition against a value
* Returns true if the condition passes, false otherwise
*/
evaluate(value) {
// Store the value in the scorer
if ('score' in this.metric) {
this.metric.score = value;
}
// Use the scorer's success check function if available
if ('successCheck' in this.metric) {
return this.metric.successCheck();
}
else {
// Fallback to default comparison (greater than or equal)
return value >= this.threshold;
}
}
/**
* Convert the condition to a plain object
*/
toJSON() {
return {
metric: this.metric.toJSON ? this.metric.toJSON() : this.metric,
};
}
}
/**
* Configuration for notifications when a rule is triggered
*/
export class NotificationConfig {
constructor(enabled = true, communicationMethods = [], emailAddresses, sendAt) {
this.enabled = enabled;
this.communicationMethods = communicationMethods;
this.emailAddresses = emailAddresses;
this.sendAt = sendAt;
}
/**
* Convert the notification config to a plain object
*/
toJSON() {
return {
enabled: this.enabled,
communication_methods: this.communicationMethods,
email_addresses: this.emailAddresses,
send_at: this.sendAt,
};
}
}
/**
* Configuration for a single rule
*/
export class Rule {
constructor(name, conditions, combine_type = 'all', description, notification, ruleId) {
this.ruleId = ruleId || uuidv4();
this.name = name;
this.description = description;
this.conditions = conditions;
this.combine_type = combine_type;
this.notification = notification;
// Validate
if (!this.conditions || this.conditions.length === 0) {
throw new Error('Conditions list cannot be empty');
}
if (this.combine_type !== 'all' && this.combine_type !== 'any') {
throw new Error('Combine type must be "all" or "any"');
}
}
/**
* Convert the rule to a plain object
*/
toJSON() {
const data = {
rule_id: this.ruleId,
name: this.name,
description: this.description,
conditions: this.conditions.map(condition => {
const conditionData = {};
// Handle the metric property
if (condition.metric) {
const metricObj = condition.metric;
// Create standardized metric representation
const metricData = {
score_type: '',
threshold: 0.0
};
// Try to use object's own serialization methods
if ('toJSON' in metricObj) {
const origData = metricObj.toJSON();
// Copy any existing fields
Object.assign(metricData, origData);
}
// Ensure required fields have values
if (!metricData.score_type && metricData.name) {
metricData.score_type = metricData.name;
}
if (!metricData.score_type) {
// Try to get score_type from different possible attributes
if ('scoreType' in metricObj) {
metricData.score_type = metricObj.scoreType;
}
else if ('name' in metricObj) {
metricData.score_type = metricObj.name;
}
else {
// Last resort: use string representation
metricData.score_type = String(metricObj);
}
}
// Make sure threshold is set
if (!metricData.threshold && metricData.threshold !== 0.0) {
if ('threshold' in metricObj) {
metricData.threshold = metricObj.threshold;
}
else {
// Use condition threshold
metricData.threshold = condition.threshold;
}
}
conditionData.metric = metricData;
}
return conditionData;
}),
combine_type: this.combine_type,
};
if (this.notification) {
data.notification = this.notification.toJSON();
}
return data;
}
}
/**
* Result of evaluating a rule
*/
export class AlertResult {
constructor(status, ruleName, conditionsResult, metadata = {}, ruleId, notification) {
this.status = status;
this.ruleId = ruleId;
this.ruleName = ruleName;
this.conditionsResult = conditionsResult;
this.metadata = metadata;
this.notification = notification;
}
/**
* Get example_id from metadata for backward compatibility
*/
get exampleId() {
return this.metadata.example_id;
}
/**
* Get timestamp from metadata for backward compatibility
*/
get timestamp() {
return this.metadata.timestamp;
}
/**
* Convert the alert result to a plain object
*/
toJSON() {
const result = {
status: this.status,
rule_name: this.ruleName,
conditions_result: this.conditionsResult,
metadata: this.metadata,
};
if (this.ruleId) {
result.rule_id = this.ruleId;
}
if (this.notification) {
result.notification = this.notification.toJSON();
}
return result;
}
}
/**
* Engine for creating and evaluating rules against metrics
*/
export class RulesEngine {
constructor(rules) {
this.rules = rules;
}
/**
* Configure notification settings for a specific rule
*/
configureNotification(ruleId, enabled = true, communicationMethods = [], emailAddresses, sendAt) {
if (!this.rules[ruleId]) {
throw new Error(`Rule with ID ${ruleId} not found`);
}
this.rules[ruleId].notification = new NotificationConfig(enabled, communicationMethods, emailAddresses, sendAt);
}
/**
* Configure notification settings for all rules
*/
configureAllNotifications(enabled = true, communicationMethods = [], emailAddresses, sendAt) {
for (const ruleId in this.rules) {
this.configureNotification(ruleId, enabled, communicationMethods, emailAddresses, sendAt);
}
}
/**
* Evaluate all rules against a set of scores
* Returns mapping of rule IDs to their alert results
*/
evaluateRules(scores, exampleMetadata) {
const results = {};
const metadata = exampleMetadata || {};
// Add timestamp if not present
if (!metadata.timestamp) {
metadata.timestamp = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15);
}
for (const ruleId in this.rules) {
const rule = this.rules[ruleId];
const conditionsResult = [];
let allPassed = true;
let anyPassed = false;
// Evaluate each condition
for (const condition of rule.conditions) {
const metricName = condition.metricName;
const value = scores[metricName] || 0;
const threshold = condition.threshold;
const passed = condition.evaluate(value);
conditionsResult.push({
metric: metricName,
value,
threshold,
passed,
});
if (!passed) {
allPassed = false;
}
else {
anyPassed = true;
}
}
// Determine if the rule is triggered based on combine type
const isTriggered = (rule.combine_type === 'all' && allPassed) ||
(rule.combine_type === 'any' && anyPassed);
results[ruleId] = new AlertResult(isTriggered ? AlertStatus.TRIGGERED : AlertStatus.NOT_TRIGGERED, rule.name, conditionsResult, metadata, rule.ruleId, rule.notification);
}
return results;
}
/**
* Evaluate all rules against multiple examples in parallel
*/
evaluateRulesParallel(exampleScores_1, exampleMetadata_1) {
return __awaiter(this, arguments, void 0, function* (exampleScores, exampleMetadata, maxConcurrent = 100) {
const results = {};
const exampleIds = Object.keys(exampleScores);
// Process in batches to control concurrency
for (let i = 0; i < exampleIds.length; i += maxConcurrent) {
const batch = exampleIds.slice(i, i + maxConcurrent);
const batchPromises = batch.map(exampleId => {
const scores = exampleScores[exampleId];
const metadata = exampleMetadata[exampleId] || {};
return this.evaluateRules(scores, metadata);
});
const batchResults = yield Promise.all(batchPromises);
batch.forEach((exampleId, index) => {
results[exampleId] = batchResults[index];
});
}
return results;
});
}
}
//# sourceMappingURL=rules.js.map