wave-roll/dist/lib/evaluation/validation/metrics-comparator.d.ts

JavaScript Library for Comparative MIDI Piano-Roll Visualization

/**
 * Metrics Comparator for mir_eval Compatibility Validation
 *
 * This utility validates that our transcription evaluation implementation
 * produces results consistent with mir_eval standards. It compares:
 *
 * 1. Note matching results (TP, FP, FN counts)
 * 2. Precision, Recall, F1 scores
 * 3. Onset, offset, and overlap metrics
 * 4. Velocity-aware evaluation results
 *
 * Reference: https://github.com/mir-evaluation/mir_eval
 */
import type { ParsedMidi } from '@/lib/midi/types';
import type { TranscriptionToleranceOptions, VelocityToleranceOptions } from '@/lib/evaluation/transcription/constants';
export interface MirEvalExpectedResults {
    /** Expected number of true positives */
    truePositives: number;
    /** Expected number of false positives */
    falsePositives: number;
    /** Expected number of false negatives */
    falseNegatives: number;
    /** Expected precision (TP / (TP + FP)) */
    precision: number;
    /** Expected recall (TP / (TP + FN)) */
    recall: number;
    /** Expected F1 score (2 * precision * recall / (precision + recall)) */
    f1Score: number;
    /** Expected average overlap ratio */
    averageOverlap?: number;
    /** Source of expected results (e.g., 'mir_eval_v0.7') */
    source: string;
    /** Additional context or notes */
    notes?: string;
}
export interface ComparisonOptions {
    /** Tolerance for floating-point metric comparisons (default: 1e-6) */
    metricTolerance: number;
    /** Whether to perform strict count validation */
    strictCounts: boolean;
    /** Whether to validate individual match details */
    validateMatches: boolean;
    /** Whether to compare velocity-aware metrics */
    includeVelocityMetrics: boolean;
}
export interface ComparisonResult {
    /** Whether all comparisons passed within tolerance */
    isCompatible: boolean;
    /** Overall compatibility score (0-1) */
    compatibilityScore: number;
    /** Detailed comparison results */
    comparisons: MetricComparison[];
    /** Summary of differences */
    summary: ComparisonSummary;
    /** Recommendations for improvement */
    recommendations: string[];
}
export interface MetricComparison {
    /** Name of the metric being compared */
    metricName: string;
    /** Expected value (from mir_eval or reference) */
    expected: number;
    /** Actual value (from our implementation) */
    actual: number;
    /** Absolute difference */
    difference: number;
    /** Relative difference (percentage) */
    relativeDifference: number;
    /** Whether the comparison passed within tolerance */
    passed: boolean;
    /** Severity of the difference */
    severity: 'acceptable' | 'concerning' | 'critical';
    /** Additional context */
    context?: string;
}
export interface ComparisonSummary {
    /** Total number of metrics compared */
    totalMetrics: number;
    /** Number of metrics that passed */
    passedMetrics: number;
    /** Number of metrics with concerning differences */
    concerningMetrics: number;
    /** Number of metrics with critical differences */
    criticalMetrics: number;
    /** Largest absolute difference observed */
    maxAbsoluteDifference: number;
    /** Largest relative difference observed */
    maxRelativeDifference: number;
    /** Average absolute difference */
    averageAbsoluteDifference: number;
}
export declare class MirEvalMetricsComparator {
    private readonly defaultOptions;
    /**
     * Compare our evaluation results with expected mir_eval results
     */
    compareWithExpected(reference: ParsedMidi, estimated: ParsedMidi, expected: MirEvalExpectedResults, toleranceOptions?: Partial<TranscriptionToleranceOptions>, velocityOptions?: Partial<VelocityToleranceOptions>, comparisonOptions?: Partial<ComparisonOptions>): ComparisonResult;
    /**
     * Compare two evaluation runs for consistency
     */
    compareEvaluationRuns(reference: ParsedMidi, estimated: ParsedMidi, toleranceOptions1: Partial<TranscriptionToleranceOptions>, toleranceOptions2: Partial<TranscriptionToleranceOptions>, runName1?: string, runName2?: string, comparisonOptions?: Partial<ComparisonOptions>): ComparisonResult;
    /**
     * Generate a comprehensive validation report
     */
    generateValidationReport(testCases: Array<{
        name: string;
        reference: ParsedMidi;
        estimated: ParsedMidi;
        expected: MirEvalExpectedResults;
        toleranceOptions?: Partial<TranscriptionToleranceOptions>;
        velocityOptions?: Partial<VelocityToleranceOptions>;
    }>, comparisonOptions?: Partial<ComparisonOptions>): ValidationReport;
    private compareMetric;
    private calculateSummary;
    private generateRecommendations;
    private aggregateSummaries;
    private generateOverallRecommendations;
}
export interface ValidationReport {
    overallCompatible: boolean;
    averageCompatibilityScore: number;
    testCaseResults: Array<{
        testCaseName: string;
        result: ComparisonResult;
    }>;
    aggregatedSummary: ComparisonSummary;
    overallRecommendations: string[];
    timestamp: string;
}
export declare const mirEvalComparator: MirEvalMetricsComparator;
/**
 * Predefined test cases with known mir_eval results
 * These can be used for regression testing and validation
 */
export declare const KNOWN_TEST_CASES: {
    PERFECT_MATCH: {
        name: string;
        expected: {
            truePositives: number;
            falsePositives: number;
            falseNegatives: number;
            precision: number;
            recall: number;
            f1Score: number;
            averageOverlap: number;
            source: string;
        };
    };
    NO_MATCH: {
        name: string;
        expected: {
            truePositives: number;
            falsePositives: number;
            falseNegatives: number;
            precision: number;
            recall: number;
            f1Score: number;
            averageOverlap: number;
            source: string;
        };
    };
    PARTIAL_MATCH: {
        name: string;
        expected: {
            truePositives: number;
            falsePositives: number;
            falseNegatives: number;
            precision: number;
            recall: number;
            f1Score: number;
            source: string;
        };
    };
};
//# sourceMappingURL=metrics-comparator.d.ts.map