@claude-flow/embeddings/dist/ledger-analyzer.js

V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking

/**
 * ADR-121 Phase 22 — Ledger analyzer + regression detection.
 *
 * The witness ledger from Phase 18 stores N historical benchmark
 * runs as a tamper-evident hash chain. Until now it served only as
 * provenance — "I observed these numbers at this commit." This phase
 * adds the missing layer: **using the chain to detect performance
 * regressions** on every new run.
 *
 * Pattern:
 *   1. Read the ledger.
 *   2. For each benchmark name, pull every historical entry that
 *      ran the same benchmark.
 *   3. Extract a named metric path (e.g. "results.summary.adaptive.meanRecallAt5").
 *   4. Compare the current value to the baseline (most recent prior
 *      entry, or the best, or a rolling window) and report the delta.
 *   5. Fail CI if the delta exceeds the configured threshold.
 *
 * Because the ledger is hash-chained (Phase 18), an attacker can't
 * silently rewrite a historical baseline to make a regression look
 * like an improvement — the chain verification would catch the
 * tamper before the regression check ran.
 */
/**
 * Extract a value at a dotted path from a nested object. Returns
 * `undefined` if any segment is missing or the final value is not
 * numeric.
 */
export function getMetricAtPath(obj, path) {
    if (path.length === 0)
        return undefined;
    const segments = path.split('.');
    let cur = obj;
    for (const seg of segments) {
        if (cur === null || cur === undefined)
            return undefined;
        if (typeof cur !== 'object')
            return undefined;
        cur = cur[seg];
    }
    return typeof cur === 'number' && Number.isFinite(cur) ? cur : undefined;
}
/**
 * Collect all ledger entries with a given benchmark name, in chain
 * order (oldest first). Useful for building plots, computing
 * rolling-window stats, etc.
 */
export function entriesForBenchmark(ledger, benchmarkName) {
    return ledger.entries.filter(e => e.benchmark === benchmarkName);
}
/**
 * Pick the baseline value from a ledger for a (benchmark, metric)
 * pair under the supplied strategy. Returns null when the chain has
 * no entries with both a matching benchmark name AND a numeric value
 * at the metric path.
 */
export function pickBaseline(ledger, benchmarkName, metricPath, selection) {
    const candidates = entriesForBenchmark(ledger, benchmarkName)
        .map(e => ({ entry: e, value: getMetricAtPath(e, metricPath) }))
        .filter((x) => x.value !== undefined);
    if (candidates.length === 0)
        return null;
    const direction = selection.direction ?? 'higher';
    switch (selection.strategy) {
        case 'first':
            return { value: candidates[0].value, entry: candidates[0].entry };
        case 'latest':
            return { value: candidates[candidates.length - 1].value, entry: candidates[candidates.length - 1].entry };
        case 'best': {
            let best = candidates[0];
            for (const c of candidates.slice(1)) {
                const isBetter = direction === 'higher' ? c.value > best.value : c.value < best.value;
                if (isBetter)
                    best = c;
            }
            return { value: best.value, entry: best.entry };
        }
        default:
            throw new Error(`unknown baseline strategy: ${selection.strategy}`);
    }
}
/**
 * Check whether the current value regressed against the ledger
 * baseline. Returns a structured result with the delta and reason.
 */
export function checkRegression(ledger, benchmarkName, metricPath, currentValue, options = {}) {
    const threshold = options.threshold ?? 0.10;
    const baselineSel = options.baseline ?? { strategy: 'latest', direction: 'higher' };
    const baseline = pickBaseline(ledger, benchmarkName, metricPath, baselineSel);
    if (baseline === null) {
        return {
            benchmark: benchmarkName,
            metricPath,
            currentValue,
            baselineValue: null,
            baselineFrom: null,
            delta: 0,
            percentChange: 0,
            passed: true,
            reason: `no prior entries for benchmark='${benchmarkName}' with numeric '${metricPath}' — first run, nothing to regress against`,
        };
    }
    const direction = baselineSel.direction ?? 'higher';
    const delta = currentValue - baseline.value;
    // Guard divide-by-zero (rare in practice — recall=0 baselines).
    const percentChange = baseline.value === 0
        ? (currentValue === 0 ? 0 : Infinity)
        : delta / Math.abs(baseline.value);
    // For higher-is-better: regression = percentChange < -threshold
    // For lower-is-better:  regression = percentChange > +threshold
    let passed;
    let reason;
    if (direction === 'higher') {
        if (percentChange >= -threshold) {
            passed = true;
            const sign = percentChange >= 0 ? '+' : '';
            reason = `current ${currentValue.toFixed(4)} ${sign}${(percentChange * 100).toFixed(1)}% vs baseline ${baseline.value.toFixed(4)} (within ${(threshold * 100).toFixed(0)}% tolerance for higher-is-better)`;
        }
        else {
            passed = false;
            reason = `REGRESSION: current ${currentValue.toFixed(4)} dropped ${Math.abs(percentChange * 100).toFixed(1)}% vs baseline ${baseline.value.toFixed(4)} (exceeds ${(threshold * 100).toFixed(0)}% threshold)`;
        }
    }
    else {
        if (percentChange <= threshold) {
            passed = true;
            const sign = percentChange >= 0 ? '+' : '';
            reason = `current ${currentValue.toFixed(4)} ${sign}${(percentChange * 100).toFixed(1)}% vs baseline ${baseline.value.toFixed(4)} (within ${(threshold * 100).toFixed(0)}% tolerance for lower-is-better)`;
        }
        else {
            passed = false;
            reason = `REGRESSION: current ${currentValue.toFixed(4)} grew ${(percentChange * 100).toFixed(1)}% vs baseline ${baseline.value.toFixed(4)} (exceeds ${(threshold * 100).toFixed(0)}% threshold)`;
        }
    }
    return {
        benchmark: benchmarkName,
        metricPath,
        currentValue,
        baselineValue: baseline.value,
        baselineFrom: { sequence: baseline.entry.sequence, timestamp: baseline.entry.timestamp },
        delta,
        percentChange,
        passed,
        reason,
    };
}
export function checkRegressionBatch(ledger, inputs) {
    const checks = inputs.map(i => checkRegression(ledger, i.benchmark, i.metricPath, i.currentValue, i.options));
    return {
        allPassed: checks.every(c => c.passed),
        checks,
    };
}
//# sourceMappingURL=ledger-analyzer.js.map