@himorishige/noren-devtools/dist/evaluation.d.ts

Development and testing tools for Noren PII detection library

/**
 * Accuracy measurement framework for PII detection evaluation
 * Streamlined implementation with unified ground truth management and metrics calculation
 */
import type { Registry } from '@himorishige/noren-core';
export interface GroundTruthAnnotation {
    start: number;
    end: number;
    type: string;
    value: string;
    confidence?: number;
    metadata?: {
        annotator?: string;
        difficulty?: 'easy' | 'medium' | 'hard';
        source?: string;
    };
}
export interface GroundTruthEntry {
    id: string;
    text: string;
    annotations: GroundTruthAnnotation[];
    metadata?: {
        source?: string;
        domain?: string;
        language?: string;
        created_at?: number;
    };
}
export interface DetectionResult {
    start: number;
    end: number;
    type: string;
    value: string;
    confidence: number;
    risk?: string;
}
export interface EvaluationResult {
    entry_id: string;
    true_positives: Array<{
        detected: DetectionResult;
        ground_truth: GroundTruthAnnotation;
        overlap_ratio: number;
    }>;
    false_positives: DetectionResult[];
    false_negatives: GroundTruthAnnotation[];
    precision: number;
    recall: number;
    f1_score: number;
}
export interface AggregateMetrics {
    total_entries: number;
    total_annotations: number;
    total_detections: number;
    true_positives: number;
    false_positives: number;
    false_negatives: number;
    precision: number;
    recall: number;
    f1_score: number;
    type_metrics: Record<string, {
        tp: number;
        fp: number;
        fn: number;
        precision: number;
        recall: number;
        f1_score: number;
    }>;
    confidence_analysis: {
        avg_tp_confidence: number;
        avg_fp_confidence: number;
        confidence_buckets: Array<{
            range: string;
            tp: number;
            fp: number;
            precision: number;
        }>;
    };
}
export interface EvaluationConfig {
    sample_size?: number;
    overlap_threshold?: number;
    confidence_threshold?: number;
    include_false_positives?: boolean;
    include_false_negatives?: boolean;
    exclude_types?: string[];
}
export declare class GroundTruthManager {
    private entries;
    addEntry(entry: GroundTruthEntry): void;
    getEntry(id: string): GroundTruthEntry | undefined;
    getAllEntries(): GroundTruthEntry[];
    getEntriesByFilter(filter: (entry: GroundTruthEntry) => boolean): GroundTruthEntry[];
    clear(): void;
    exportToJson(): string;
    importFromJson(jsonData: string): void;
}
export declare class EvaluationEngine {
    private groundTruthManager?;
    private config?;
    constructor(groundTruthManager?: GroundTruthManager | undefined, config?: EvaluationConfig | undefined);
    evaluateEntry(entryId: string, detections: DetectionResult[], config?: EvaluationConfig): EvaluationResult;
    evaluateAgainstGroundTruth(registry: Registry, groundTruthManager: GroundTruthManager, config?: EvaluationConfig): Promise<{
        aggregate: AggregateMetrics;
        results: EvaluationResult[];
    }>;
    private evaluateEntry_internal;
    private aggregateResults;
    private calculateOverlap;
    private printEvaluationReport;
}
export declare function createSyntheticEntry(id: string, patterns: Array<{
    type: string;
    pattern: string;
}>): GroundTruthEntry;
export declare function createEmailTestDataset(): GroundTruthManager;