@claude-flow/embeddings/dist/hybrid-retrieval.js

V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking

/**
 * ADR-121 Phase 17 — Hybrid sparse+dense retrieval.
 *
 * Composes BM25 (sparse lexical) + dense vector search via RRF
 * (Phase 11) into a single fused ranking. The standard production
 * pattern for catching documents with rare/technical terms that
 * dense embeddings underweight while preserving the semantic
 * matching that BM25 misses.
 *
 * The two signals are not score-comparable directly (BM25 is
 * unbounded positive, cosine ∈ [-1, 1]) — RRF operates on ranks,
 * which makes it the right fusion primitive here.
 *
 * Pipeline:
 *   1. denseHits = denseTopK(corpus, queryVec, fetchK)
 *   2. sparseHits = bm25TopK(queryText, bm25Index, fetchK)
 *   3. fused = RRF([denseHits, sparseHits], { k, kRrf, listWeights })
 *
 * Caller provides:
 *   - the dense search function (any backing: AnnRouter, in-memory,
 *     DiskANN snapshot)
 *   - the BM25 index (built once per corpus via buildBm25Index)
 *
 * Composable with all earlier primitives: HyDE produces the query
 * vector, MMR-rerank the fused output, etc.
 */
import { reciprocalRankFusion } from './rrf.js';
import { bm25TopK } from './bm25.js';
/**
 * Run hybrid sparse+dense retrieval. Returns RRF-fused hits plus
 * diagnostics for observability.
 *
 * The dense search is async (matches real backings); the BM25 search
 * is sync (pure in-memory). Caller passes the dense search function
 * so this module stays decoupled from any specific ANN backing.
 */
export async function hybridRetrieval(queryText, queryVector, bm25Index, denseSearch, options) {
    const k = options.k;
    if (!Number.isInteger(k) || k < 1) {
        throw new Error('hybridRetrieval: k must be a positive integer');
    }
    const fetchK = options.fetchK ?? Math.max(k * 3, k);
    const kRrf = options.kRrf ?? 60;
    const listWeights = options.listWeights ?? [1, 1];
    if (listWeights.length !== 2) {
        throw new Error('hybridRetrieval: listWeights must be [denseWeight, sparseWeight]');
    }
    // Run both signals in parallel.
    const [denseHits, sparseHits] = await Promise.all([
        Promise.resolve(denseSearch(queryVector, fetchK)),
        Promise.resolve(bm25TopK(queryText, bm25Index, fetchK)),
    ]);
    // Compute overlap diagnostic before fusion.
    const denseIds = new Set(denseHits.map(h => h.id));
    const overlapIds = sparseHits.filter(h => denseIds.has(h.id)).map(h => h.id);
    const meanSparseScore = sparseHits.length === 0
        ? 0
        : sparseHits.reduce((s, h) => s + h.score, 0) / sparseHits.length;
    // RRF-fuse. Annotate payloads with the source list so the caller
    // can see which signal carried each hit.
    const denseRrfList = denseHits.map(h => ({ id: h.id, payload: { source: 'dense', score: h.score } }));
    const sparseRrfList = sparseHits.map(h => ({ id: h.id, payload: { source: 'sparse', score: h.score } }));
    const fused = reciprocalRankFusion([denseRrfList, sparseRrfList], {
        k,
        kRrf,
        listWeights,
    });
    return {
        hits: fused,
        diagnostics: {
            denseHitCount: denseHits.length,
            sparseHitCount: sparseHits.length,
            overlapIds,
            meanSparseScore,
        },
    };
}
//# sourceMappingURL=hybrid-retrieval.js.map