@claude-flow/embeddings/src/hybrid-retrieval.ts

V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking

/**
 * ADR-121 Phase 17 — Hybrid sparse+dense retrieval.
 *
 * Composes BM25 (sparse lexical) + dense vector search via RRF
 * (Phase 11) into a single fused ranking. The standard production
 * pattern for catching documents with rare/technical terms that
 * dense embeddings underweight while preserving the semantic
 * matching that BM25 misses.
 *
 * The two signals are not score-comparable directly (BM25 is
 * unbounded positive, cosine ∈ [-1, 1]) — RRF operates on ranks,
 * which makes it the right fusion primitive here.
 *
 * Pipeline:
 *   1. denseHits = denseTopK(corpus, queryVec, fetchK)
 *   2. sparseHits = bm25TopK(queryText, bm25Index, fetchK)
 *   3. fused = RRF([denseHits, sparseHits], { k, kRrf, listWeights })
 *
 * Caller provides:
 *   - the dense search function (any backing: AnnRouter, in-memory,
 *     DiskANN snapshot)
 *   - the BM25 index (built once per corpus via buildBm25Index)
 *
 * Composable with all earlier primitives: HyDE produces the query
 * vector, MMR-rerank the fused output, etc.
 */

import { reciprocalRankFusion, type RrfFusedHit } from './rrf.js';
import { bm25TopK, type Bm25Index } from './bm25.js';

export interface HybridRetrievalOptions {
  /** Final number of fused results to return. */
  readonly k: number;
  /**
   * Top-N to pull from each sub-retriever (BM25 + dense) before RRF.
   * Default 3*k. Larger = wider input pool, more compute.
   */
  readonly fetchK?: number;
  /** RRF smoothing constant (Cormack-Clarke-Büttcher default 60). */
  readonly kRrf?: number;
  /**
   * Per-list weights for RRF, in [dense_weight, sparse_weight] order.
   * Default [1, 1]. Use to bias toward one signal — e.g. [1.5, 1.0]
   * to prefer dense semantic matches in general while letting BM25
   * still contribute on rare-term docs.
   */
  readonly listWeights?: readonly [number, number];
}

export interface HybridRetrievalDiagnostics {
  readonly denseHitCount: number;
  readonly sparseHitCount: number;
  /** IDs that appeared in BOTH lists (overlap signal). */
  readonly overlapIds: ReadonlyArray<string>;
  /** Mean BM25 score across the sparse hits, for cost attribution. */
  readonly meanSparseScore: number;
}

export interface HybridRetrievalResult {
  readonly hits: ReadonlyArray<RrfFusedHit>;
  readonly diagnostics: HybridRetrievalDiagnostics;
}

/**
 * Run hybrid sparse+dense retrieval. Returns RRF-fused hits plus
 * diagnostics for observability.
 *
 * The dense search is async (matches real backings); the BM25 search
 * is sync (pure in-memory). Caller passes the dense search function
 * so this module stays decoupled from any specific ANN backing.
 */
export async function hybridRetrieval(
  queryText: string,
  queryVector: Float32Array | number[],
  bm25Index: Bm25Index,
  denseSearch: (queryVec: Float32Array | number[], k: number) => Promise<ReadonlyArray<{ id: string; score?: number }>> | ReadonlyArray<{ id: string; score?: number }>,
  options: HybridRetrievalOptions,
): Promise<HybridRetrievalResult> {
  const k = options.k;
  if (!Number.isInteger(k) || k < 1) {
    throw new Error('hybridRetrieval: k must be a positive integer');
  }
  const fetchK = options.fetchK ?? Math.max(k * 3, k);
  const kRrf = options.kRrf ?? 60;
  const listWeights = options.listWeights ?? [1, 1];
  if (listWeights.length !== 2) {
    throw new Error('hybridRetrieval: listWeights must be [denseWeight, sparseWeight]');
  }

  // Run both signals in parallel.
  const [denseHits, sparseHits] = await Promise.all([
    Promise.resolve(denseSearch(queryVector, fetchK)),
    Promise.resolve(bm25TopK(queryText, bm25Index, fetchK)),
  ]);

  // Compute overlap diagnostic before fusion.
  const denseIds = new Set(denseHits.map(h => h.id));
  const overlapIds = sparseHits.filter(h => denseIds.has(h.id)).map(h => h.id);
  const meanSparseScore = sparseHits.length === 0
    ? 0
    : sparseHits.reduce((s, h) => s + h.score, 0) / sparseHits.length;

  // RRF-fuse. Annotate payloads with the source list so the caller
  // can see which signal carried each hit.
  const denseRrfList = denseHits.map(h => ({ id: h.id, payload: { source: 'dense', score: h.score } }));
  const sparseRrfList = sparseHits.map(h => ({ id: h.id, payload: { source: 'sparse', score: h.score } }));

  const fused = reciprocalRankFusion([denseRrfList, sparseRrfList], {
    k,
    kRrf,
    listWeights,
  });

  return {
    hits: fused,
    diagnostics: {
      denseHitCount: denseHits.length,
      sparseHitCount: sparseHits.length,
      overlapIds,
      meanSparseScore,
    },
  };
}