@claude-flow/embeddings/dist/bm25.d.ts

V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking

/**
 * ADR-121 Phase 17 — Okapi BM25 (sparse lexical retrieval).
 *
 * Dense vector retrieval (cosine over learned embeddings) is what
 * Phases 9-16 ship. It's powerful but has a known failure mode:
 * rare or technical terms that the embedding model never saw at
 * training time get averaged out in the dense vector, so docs that
 * literally contain "CVE-2024-12345" or "sha256:abc…" rank poorly
 * even when they're the exact match.
 *
 * The standard production fix is **hybrid retrieval**: combine a
 * sparse lexical signal (BM25 — what classical search engines use)
 * with the dense signal (cosine), then RRF-fuse the two ranked
 * lists. The sparse signal catches the rare-term docs the dense
 * signal underweights; the dense signal catches the semantic-only
 * matches the sparse signal misses.
 *
 * This module ships pure-function BM25 (Okapi variant — the modern
 * production default since Robertson-Walker 1994) with no external
 * deps. Composes with the Phase 11 RRF primitive for hybrid retrieval.
 *
 *   score(q, d) = Σ_{t in q} IDF(t) · (tf(t,d) · (k1 + 1)) / (tf(t,d) + k1·(1 - b + b·|d|/avgdl))
 *
 *   IDF(t) = ln((N - df(t) + 0.5) / (df(t) + 0.5) + 1)
 *
 * Parameters:
 *   k1 ∈ [1.2, 2.0] — term frequency saturation (default 1.5)
 *   b  ∈ [0.0, 1.0] — document-length normalization (default 0.75)
 */
export interface Bm25Options {
    /** Term frequency saturation. Default 1.5 (Okapi standard). */
    readonly k1?: number;
    /** Length normalization. Default 0.75. */
    readonly b?: number;
}
export interface Bm25Document {
    readonly id: string;
    readonly text: string;
}
export interface Bm25Index {
    readonly docs: ReadonlyArray<{
        id: string;
        tokens: string[];
        length: number;
    }>;
    /** Document frequency: token → number of docs containing it. */
    readonly df: ReadonlyMap<string, number>;
    /** Number of documents. */
    readonly N: number;
    /** Average document length. */
    readonly avgdl: number;
    readonly k1: number;
    readonly b: number;
}
export interface Bm25Hit {
    readonly id: string;
    readonly score: number;
}
/**
 * Default tokenizer: lowercase, split on non-alphanumeric, drop
 * empty strings, drop tokens of length < 2 (stopword-ish noise).
 *
 * Production callers can supply their own tokenizer (e.g. stemmer,
 * stop-word filter, n-grams) by tokenizing themselves and passing
 * the pre-tokenized form via `buildBm25IndexFromTokens`.
 */
export declare function tokenize(text: string): string[];
/**
 * Build a BM25 index over a corpus. O(N · avg-doc-length).
 *
 * The index is immutable — to add docs, build a new index. For
 * streaming corpora the right move is to maintain the (df, N, avgdl,
 * docs) structures explicitly and rebuild incrementally; this
 * module keeps it simple.
 */
export declare function buildBm25Index(corpus: ReadonlyArray<Bm25Document>, options?: Bm25Options): Bm25Index;
/**
 * Score a single document against a query under the index. Used
 * by bm25TopK; exported for callers who want to score a specific
 * (query, doc) pair.
 */
export declare function bm25Score(queryTokens: ReadonlyArray<string>, doc: {
    tokens: string[];
    length: number;
}, index: Bm25Index): number;
/**
 * Top-k BM25 search. Returns docs with score > 0 (docs that share
 * at least one token with the query), sorted descending. Ties
 * resolve by id ascending for determinism.
 *
 * Time complexity: O(N · |q|) where N = doc count, |q| = unique
 * query tokens.
 */
export declare function bm25TopK(query: string, index: Bm25Index, k: number): Bm25Hit[];
/**
 * Compute the inverse document frequency for a single token. Returns
 * 0 for tokens not in the corpus. Useful for the adaptive router's
 * rare-token-density signal.
 */
export declare function idfOf(token: string, index: Bm25Index): number;
/**
 * Rare-token density of a query relative to a BM25 index. Returns
 * the **mean IDF** of unique tokens in the query that appear in the
 * corpus at least once (tokens not in the corpus would have IDF 0
 * and would dilute the signal — they don't represent "rare
 * technical terms our retrieval can act on", they represent
 * "tokens our corpus doesn't have").
 *
 * Used by the adaptive router as the hybrid-retrieval signal: high
 * mean IDF means the query has rare/technical terms that dense
 * cosine retrieval tends to underweight.
 */
export declare function queryMeanIdf(query: string, index: Bm25Index): number;
//# sourceMappingURL=bm25.d.ts.map