@claude-flow/embeddings/src/embedding-fusion.ts

V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking

/**
 * ADR-121 Phase 12 — Embedding-level fusion (HyDE).
 *
 * HyDE (Hypothetical Document Embeddings — Gao, Ma, Lin, Callan 2022,
 * "Precise Zero-Shot Dense Retrieval without Relevance Labels") fixes
 * a structural problem with question-based dense retrieval: question
 * embeddings live in "question space" while documents embed into
 * "answer space", so cosine similarity systematically underweights
 * relevant docs whose surface form differs from the question.
 *
 * The HyDE recipe:
 *   1. Have an LLM generate N hypothetical answers to the question.
 *   2. Embed each hypothetical answer.
 *   3. Average those embeddings into a single query vector.
 *   4. Search with the averaged vector.
 *
 * Empirically: this single transform produces SOTA zero-shot
 * retrieval on BEIR — beating supervised in-domain dense retrievers
 * on many tasks. The trick works because hypothetical answers live
 * in the same answer-space as the corpus, so the averaged vector
 * lands near the true relevant docs.
 *
 * This module ships the **averaging** step as a pure function — the
 * LLM-generation step is the caller's job (the orchestrator agent
 * already has an LLM). Composes with every existing retrieval shape:
 *
 *   HyDE + plain search    = embeddings_search_text_hyde
 *   HyDE + MMR rerank      = caller-composable (search_text_hyde
 *                            results piped through mmrRerank)
 *   HyDE + RRF ensemble    = caller-composable (search_text_hyde
 *                            across multiple LLM samples, fused)
 *
 * Relationship to RRF (Phase 11):
 *   - RRF fuses at the **rank** level: N searches, merge ranks. More
 *     expensive but preserves intent boundaries between variants.
 *   - HyDE/avg fuses at the **embedding** level: 1 search after vector
 *     average. Cheaper but interpolates between variants — the
 *     averaged vector represents the "centroid" intent.
 *
 *   The two are complementary, not redundant. Production systems
 *   often use HyDE inside one ranked list, and RRF across multiple
 *   ranked lists (one HyDE-search, one BM25-search, etc.).
 */

export interface AverageEmbeddingsOptions {
  /**
   * Per-vector weights. Length must equal vectors.length.
   * Defaults to uniform = 1 each.
   *
   * Use weights when the caller has confidence signals — e.g. weight
   * the user's original question 0.5× and the LLM's hypothetical
   * answers 1.0× each, so the answers dominate (the HyDE paper's
   * default recipe).
   */
  readonly weights?: ReadonlyArray<number>;
  /**
   * L2-normalize each input vector BEFORE averaging. Default true.
   *
   * This is the standard HyDE recipe — without it, vectors with
   * larger norms dominate the average regardless of their semantic
   * weight. Set to false only if you already pre-normalized
   * (cheaper) or if you specifically want norm-weighted averaging.
   */
  readonly normalizeInputs?: boolean;
  /**
   * L2-normalize the OUTPUT averaged vector. Default true.
   *
   * Required for cosine-similarity search backings (HNSW with
   * cosine, RaBitQ, DiskANN). The math: avg of unit vectors is not
   * itself a unit vector.
   */
  readonly normalizeOutput?: boolean;
}

/**
 * Average N embedding vectors into a single embedding vector.
 *
 * Time complexity: O(N · dim). Memory: O(dim).
 *
 * Returns a brand-new Float32Array — does not mutate inputs.
 */
export function averageEmbeddings(
  vectors: ReadonlyArray<Float32Array | number[]>,
  options: AverageEmbeddingsOptions = {},
): Float32Array {
  if (!Array.isArray(vectors) || vectors.length === 0) {
    throw new Error('averageEmbeddings: vectors must be a non-empty array');
  }
  const normalizeInputs = options.normalizeInputs ?? true;
  const normalizeOutput = options.normalizeOutput ?? true;
  const weights = options.weights ?? vectors.map(() => 1);
  if (weights.length !== vectors.length) {
    throw new Error(
      `averageEmbeddings: weights.length (${weights.length}) must match vectors.length (${vectors.length})`,
    );
  }
  for (let i = 0; i < weights.length; i++) {
    if (weights[i]! < 0) {
      throw new Error(`averageEmbeddings: weights[${i}] is negative (${weights[i]})`);
    }
  }

  // Find dim from the first vector + assert all match.
  const first = vectors[0]!;
  const dim = first.length;
  for (let i = 1; i < vectors.length; i++) {
    if (vectors[i]!.length !== dim) {
      throw new Error(
        `averageEmbeddings: vector ${i} has dim ${vectors[i]!.length}, expected ${dim}`,
      );
    }
  }

  const out = new Float32Array(dim);
  let totalWeight = 0;
  for (let i = 0; i < vectors.length; i++) {
    const v = vectors[i]!;
    const w = weights[i]!;
    if (w === 0) continue;
    totalWeight += w;

    let norm = 1;
    if (normalizeInputs) {
      let sq = 0;
      for (let j = 0; j < dim; j++) sq += v[j]! * v[j]!;
      norm = sq > 0 ? Math.sqrt(sq) : 1;
    }
    const scale = w / norm;
    for (let j = 0; j < dim; j++) {
      out[j]! += v[j]! * scale;
    }
  }

  if (totalWeight === 0) {
    // All-zero-weight case — produce a deterministic zero vector
    // rather than NaN.
    return out;
  }

  // Mean — divide by total weight.
  for (let j = 0; j < dim; j++) {
    out[j]! /= totalWeight;
  }

  if (normalizeOutput) {
    let sq = 0;
    for (let j = 0; j < dim; j++) sq += out[j]! * out[j]!;
    if (sq > 0) {
      const inv = 1 / Math.sqrt(sq);
      for (let j = 0; j < dim; j++) out[j]! *= inv;
    }
  }
  return out;
}

/**
 * Convenience: confirm a Float32Array is L2-unit-normalized within
 * `tolerance`. Useful for tests + the smoke driver to verify the
 * normalizeOutput contract.
 */
export function isUnitNorm(v: Float32Array, tolerance = 1e-5): boolean {
  let sq = 0;
  for (let i = 0; i < v.length; i++) sq += v[i]! * v[i]!;
  return Math.abs(Math.sqrt(sq) - 1) < tolerance;
}