@claude-flow/embeddings
Version:
V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking
223 lines • 6.76 kB
JavaScript
/**
* ADR-121 Phase 13 — Standard IR evaluation metrics.
*
* These are the metrics every retrieval benchmark reports: recall@k,
* precision@k, mean reciprocal rank (MRR), and normalized discounted
* cumulative gain (nDCG@k). Used by the RAG benchmark harness to
* compare the 5 search_text_* primitives on the same corpus.
*
* Definitions (standard IR textbook):
*
* recall@k = |relevant ∩ retrieved_top_k| / |relevant|
* precision@k = |relevant ∩ retrieved_top_k| / k
* reciprocal rank = 1 / rank_of_first_relevant (0 if none in top-k)
* MRR = mean of reciprocal ranks over a query set
*
* DCG@k = Σ_{i=1..k} rel_i / log2(i + 1) (binary or graded)
* IDCG@k = DCG@k of the ideal ordering (relevant items first)
* nDCG@k = DCG@k / IDCG@k (0..1; 1 = perfect ordering)
*
* Inputs are intentionally generic: each metric takes a retrieved-id
* list and either a relevant-id set (binary judgements) or a
* id→relevance-grade map (graded judgements). No coupling to the
* retrieval tools — anything that produces a ranked list of ids can
* be evaluated.
*/
function toSet(rel) {
if (rel instanceof Set)
return rel;
return new Set(rel);
}
function gradeOf(rel, id) {
if (rel instanceof Map)
return rel.get(id) ?? 0;
return rel[id] ?? 0;
}
/**
* recall@k = how much of the relevant set we found in the top-k.
* Returns 0 when relevant is empty (no signal to recall).
*/
export function recallAtK(retrieved, relevant, k) {
const relSet = toSet(relevant);
if (relSet.size === 0)
return 0;
const topK = retrieved.slice(0, Math.max(0, k));
let hits = 0;
for (const id of topK)
if (relSet.has(id))
hits++;
return hits / relSet.size;
}
/**
* precision@k = what fraction of the top-k is relevant. Returns 0
* for k<=0.
*/
export function precisionAtK(retrieved, relevant, k) {
if (k <= 0)
return 0;
const relSet = toSet(relevant);
const topK = retrieved.slice(0, k);
let hits = 0;
for (const id of topK)
if (relSet.has(id))
hits++;
return hits / k;
}
/**
* Reciprocal rank: 1 / (1-based rank of first relevant item in the
* retrieved list). 0 if no relevant item appears at all.
*/
export function reciprocalRank(retrieved, relevant) {
const relSet = toSet(relevant);
for (let i = 0; i < retrieved.length; i++) {
if (relSet.has(retrieved[i]))
return 1 / (i + 1);
}
return 0;
}
/**
* Mean reciprocal rank over a set of (retrieved, relevant) pairs.
* The standard MRR over a query set.
*/
export function meanReciprocalRank(queries) {
if (queries.length === 0)
return 0;
let sum = 0;
for (const q of queries)
sum += reciprocalRank(q.retrieved, q.relevant);
return sum / queries.length;
}
/**
* DCG@k over GRADED judgements:
* DCG@k = Σ_{i=1..k} rel_i / log2(i + 1)
* Treats missing ids as relevance 0.
*/
export function dcgAtK(retrieved, graded, k) {
if (k <= 0)
return 0;
const topK = retrieved.slice(0, k);
let dcg = 0;
for (let i = 0; i < topK.length; i++) {
const g = gradeOf(graded, topK[i]);
if (g > 0)
dcg += g / Math.log2(i + 2); // i+2 because rank = i+1, denom = log2(rank+1)
}
return dcg;
}
/**
* IDCG@k — ideal DCG: assumes the top-k contains the k highest-graded
* items from the judgement pool.
*/
export function idcgAtK(graded, k) {
if (k <= 0)
return 0;
const grades = [];
if (graded instanceof Map) {
for (const v of graded.values())
if (v > 0)
grades.push(v);
}
else {
for (const v of Object.values(graded))
if (v > 0)
grades.push(v);
}
grades.sort((a, b) => b - a);
const top = grades.slice(0, k);
let idcg = 0;
for (let i = 0; i < top.length; i++) {
idcg += top[i] / Math.log2(i + 2);
}
return idcg;
}
/**
* nDCG@k = DCG@k / IDCG@k, clamped to [0, 1]. Returns 0 when no
* relevant items exist (IDCG is 0).
*
* Supports both binary judgements (pass a Set/Array → grade=1 each)
* and graded judgements (pass a Map/Record).
*/
export function ndcgAtK(retrieved, judgements, k) {
if (k <= 0)
return 0;
// Coerce binary judgements to graded form.
let graded;
if (judgements instanceof Set || Array.isArray(judgements)) {
const m = new Map();
for (const id of judgements)
m.set(id, 1);
graded = m;
}
else {
graded = judgements;
}
const idcg = idcgAtK(graded, k);
if (idcg === 0)
return 0;
return Math.min(1, dcgAtK(retrieved, graded, k) / idcg);
}
export function compareRankings(retrieved, judgements, ks) {
// Coerce judgements into a usable form for both binary + graded paths.
let relSet;
let graded;
if (judgements instanceof Set) {
relSet = judgements;
const m = new Map();
for (const id of relSet)
m.set(id, 1);
graded = m;
}
else if (Array.isArray(judgements)) {
relSet = new Set(judgements);
const m = new Map();
for (const id of relSet)
m.set(id, 1);
graded = m;
}
else {
graded = judgements;
relSet = new Set();
if (graded instanceof Map) {
for (const [id, g] of graded)
if (g > 0)
relSet.add(id);
}
else {
for (const [id, g] of Object.entries(graded)) {
if (g > 0)
relSet.add(id);
}
}
}
const recall = {};
const precision = {};
const ndcg = {};
for (const k of ks) {
recall[k] = recallAtK(retrieved, relSet, k);
precision[k] = precisionAtK(retrieved, relSet, k);
ndcg[k] = ndcgAtK(retrieved, graded, k);
}
const mrr = reciprocalRank(retrieved, relSet);
return { recall, precision, ndcg, mrr };
}
/**
* Convenience: roll up a metric over a query set (mean across queries).
* Skips queries with empty relevant sets (no signal).
*/
export function meanMetric(queries, metric) {
if (queries.length === 0)
return 0;
let sum = 0;
let count = 0;
for (const q of queries) {
const relSize = q.relevant instanceof Set || Array.isArray(q.relevant)
? q.relevant.size ?? q.relevant.length
: q.relevant instanceof Map ? q.relevant.size : Object.keys(q.relevant).length;
if (relSize === 0)
continue;
sum += metric(q.retrieved, q.relevant);
count++;
}
return count === 0 ? 0 : sum / count;
}
//# sourceMappingURL=ir-metrics.js.map