rag-cli-tester/dist/metrics/brdr-metric.js

A lightweight CLI tool for testing RAG (Retrieval-Augmented Generation) systems with different embedding combinations

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.BRDRMetric = void 0;
class BRDRMetric {
    constructor() {
        this.bankingKeywords = new Set([
            // Core banking terms
            'risk', 'capital', 'compliance', 'regulation', 'supervision', 'governance',
            'asset', 'liability', 'liquidity', 'credit', 'operational', 'market',
            'basel', 'stress', 'scenario', 'framework', 'guideline', 'standard',
            // Regulatory frameworks
            'basel iii', 'basel iv', 'dodd-frank', 'sox', 'gdpr', 'ccar', 'dfast',
            'liquidity coverage ratio', 'net stable funding ratio', 'leverage ratio',
            // Risk categories
            'credit risk', 'market risk', 'operational risk', 'liquidity risk',
            'interest rate risk', 'currency risk', 'reputation risk', 'strategic risk',
            // Compliance terms
            'aml', 'kyc', 'cdd', 'edd', 'sanctions', 'embargo', 'corruption',
            'bribery', 'fraud', 'insider trading', 'market manipulation'
        ]);
        this.conceptTerms = new Set([
            // Management and control
            'management', 'assessment', 'monitoring', 'reporting', 'measurement',
            'control', 'process', 'procedure', 'methodology', 'approach',
            'requirement', 'obligation', 'responsibility', 'accountability',
            // Financial instruments
            'derivative', 'swap', 'option', 'future', 'forward', 'bond', 'equity',
            'securitization', 'collateral', 'guarantee', 'insurance', 'hedge',
            // Organizational structure
            'board', 'committee', 'audit', 'risk committee', 'compliance officer',
            'chief risk officer', 'internal audit', 'external audit', 'regulator'
        ]);
        this.regulatoryTerms = new Set([
            'requirement', 'mandatory', 'obligatory', 'compulsory', 'enforcement',
            'penalty', 'fine', 'sanction', 'violation', 'breach', 'non-compliance',
            'deadline', 'due date', 'effective date', 'implementation', 'transition'
        ]);
    }
    calculate(expected, actual, similarity) {
        const expectedLower = expected.toLowerCase();
        const actualLower = actual.toLowerCase();
        // Calculate keyword overlap (optimized with Set operations)
        const keywordMatch = this.calculateKeywordMatch(expectedLower, actualLower);
        // Calculate concept overlap
        const conceptMatch = this.calculateConceptMatch(expectedLower, actualLower);
        // Calculate regulatory compliance score
        const regulatoryCompliance = this.calculateRegulatoryCompliance(expectedLower, actualLower);
        // Use embedding similarity as contextual relevance
        const contextualRelevance = Math.max(0, (similarity + 1) / 2);
        // Calculate semantic accuracy based on text similarity
        const semanticAccuracy = this.calculateSemanticAccuracy(expectedLower, actualLower);
        // Weighted combination for overall score
        const overallScore = (keywordMatch * 0.25 +
            conceptMatch * 0.25 +
            regulatoryCompliance * 0.20 +
            contextualRelevance * 0.20 +
            semanticAccuracy * 0.10);
        // Calculate confidence based on data quality
        const confidence = this.calculateConfidence(expectedLower, actualLower, similarity);
        return {
            overallScore: Math.min(1.0, Math.max(0, overallScore)),
            keywordMatch,
            conceptMatch,
            contextualRelevance,
            regulatoryCompliance,
            semanticAccuracy,
            confidence
        };
    }
    calculateKeywordMatch(expected, actual) {
        const expectedKeywords = this.extractKeywords(expected);
        const actualKeywords = this.extractKeywords(actual);
        if (expectedKeywords.size === 0 && actualKeywords.size === 0)
            return 1.0;
        if (expectedKeywords.size === 0 || actualKeywords.size === 0)
            return 0.0;
        const intersection = new Set([...expectedKeywords].filter(x => actualKeywords.has(x)));
        const union = new Set([...expectedKeywords, ...actualKeywords]);
        return intersection.size / union.size;
    }
    calculateConceptMatch(expected, actual) {
        const expectedConcepts = this.extractConcepts(expected);
        const actualConcepts = this.extractConcepts(actual);
        if (expectedConcepts.size === 0 && actualConcepts.size === 0)
            return 1.0;
        if (expectedConcepts.size === 0 || actualConcepts.size === 0)
            return 0.0;
        const intersection = new Set([...expectedConcepts].filter(x => actualConcepts.has(x)));
        const union = new Set([...expectedConcepts, ...actualConcepts]);
        return intersection.size / union.size;
    }
    calculateRegulatoryCompliance(expected, actual) {
        const expectedRegulatory = this.extractRegulatoryTerms(expected);
        const actualRegulatory = this.extractRegulatoryTerms(actual);
        if (expectedRegulatory.size === 0 && actualRegulatory.size === 0)
            return 1.0;
        if (expectedRegulatory.size === 0 || actualRegulatory.size === 0)
            return 0.0;
        const intersection = new Set([...expectedRegulatory].filter(x => actualRegulatory.has(x)));
        const union = new Set([...expectedRegulatory, ...actualRegulatory]);
        return intersection.size / union.size;
    }
    calculateSemanticAccuracy(expected, actual) {
        // Simple text similarity using word overlap
        const expectedWords = new Set(expected.split(/\s+/).filter(word => word.length > 2));
        const actualWords = new Set(actual.split(/\s+/).filter(word => word.length > 2));
        if (expectedWords.size === 0 && actualWords.size === 0)
            return 1.0;
        if (expectedWords.size === 0 || actualWords.size === 0)
            return 0.0;
        const intersection = new Set([...expectedWords].filter(x => actualWords.has(x)));
        const union = new Set([...expectedWords, ...actualWords]);
        return intersection.size / union.size;
    }
    calculateConfidence(expected, actual, similarity) {
        // Higher confidence for longer, more detailed texts
        const expectedLength = expected.length;
        const actualLength = actual.length;
        const lengthFactor = Math.min(1.0, Math.min(expectedLength, actualLength) / 100);
        const similarityFactor = Math.max(0, (similarity + 1) / 2);
        return (lengthFactor * 0.4 + similarityFactor * 0.6);
    }
    extractKeywords(text) {
        const keywords = new Set();
        for (const keyword of this.bankingKeywords) {
            if (text.includes(keyword)) {
                keywords.add(keyword);
            }
        }
        return keywords;
    }
    extractConcepts(text) {
        const concepts = new Set();
        for (const concept of this.conceptTerms) {
            if (text.includes(concept)) {
                concepts.add(concept);
            }
        }
        return concepts;
    }
    extractRegulatoryTerms(text) {
        const terms = new Set();
        for (const term of this.regulatoryTerms) {
            if (text.includes(term)) {
                terms.add(term);
            }
        }
        return terms;
    }
    getName() {
        return 'BRDR';
    }
    getDescription() {
        return 'Banking Regulation-specific metric that evaluates regulatory compliance, keyword matching, and concept understanding for financial knowledge bases';
    }
}
exports.BRDRMetric = BRDRMetric;
//# sourceMappingURL=brdr-metric.js.map