@claude-flow/embeddings/dist/synthetic-corpus.js

V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking

/**
 * ADR-121 Phase 15 — Synthetic topic-clustered corpus builder.
 *
 * Generates a deterministic, reproducible vector corpus with known
 * topic structure + known relevance judgements. Used by the RAG
 * scale benchmark and available as a pure-function utility for
 * users who want to benchmark their own retrieval pipelines without
 * sourcing real corpora.
 *
 * Topology: N topics × D docs per topic. Each topic has a "centroid"
 * direction in vector space; its docs are noisy variants near that
 * centroid. A configurable fraction of each topic's docs are
 * "duplicate-heavy" (very close to the centroid) — useful for
 * stressing diversification primitives. The remaining docs spread
 * along secondary axes for diversity.
 *
 * Determinism: same options → same corpus, bit-for-bit. Uses a
 * deterministic PRNG seeded by `seed` (default 42).
 */
function mulberry32(seed) {
    let s = seed >>> 0;
    return () => {
        s |= 0;
        s = (s + 0x6D2B79F5) | 0;
        let t = Math.imul(s ^ (s >>> 15), 1 | s);
        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
    };
}
function l2Normalize(v) {
    let sq = 0;
    for (let i = 0; i < v.length; i++)
        sq += v[i] * v[i];
    if (sq === 0)
        return v;
    const inv = 1 / Math.sqrt(sq);
    const out = new Float32Array(v.length);
    for (let i = 0; i < v.length; i++)
        out[i] = v[i] * inv;
    return out;
}
/**
 * Build a synthetic topic-clustered corpus.
 *
 * Time complexity: O(topics · docsPerTopic · dim). Memory: same.
 *
 * For each topic t in [0, topics):
 *   - centroid = unit vector with axis t = 1
 *   - dup docs: centroid + small noise (within duplicateRatio of total)
 *   - diverse docs: centroid + larger secondary-axis components + noise
 *   - all L2-normalized
 *
 * Returns:
 *   - entries: flat list of docs (interleaved by topic for shuffle resistance)
 *   - relevantByTopic[t]: set of doc IDs belonging to topic t
 *   - topicQueryVectors[t]: canonical query vector for topic t
 */
export function buildSyntheticTopicCorpus(options) {
    const topics = options.topics;
    const docsPerTopic = options.docsPerTopic;
    const dim = options.dim;
    const duplicateRatio = Math.max(0, Math.min(1, options.duplicateRatio ?? 0.4));
    const noiseLevel = options.noiseLevel ?? 0.05;
    const seed = options.seed ?? 42;
    if (topics <= 0)
        throw new Error('topics must be > 0');
    if (docsPerTopic <= 0)
        throw new Error('docsPerTopic must be > 0');
    if (dim < topics)
        throw new Error(`dim (${dim}) must be >= topics (${topics})`);
    const rng = mulberry32(seed);
    const dupCount = Math.round(docsPerTopic * duplicateRatio);
    const divCount = docsPerTopic - dupCount;
    const entries = [];
    const relevantByTopic = Array.from({ length: topics }, () => new Set());
    const topicQueryVectors = [];
    // Build canonical query vector per topic (one-hot on topic axis).
    for (let t = 0; t < topics; t++) {
        const q = new Float32Array(dim);
        q[t] = 1;
        topicQueryVectors.push(l2Normalize(q));
    }
    // Generate docs interleaved (round-robin by topic) so the entries
    // array isn't pre-grouped — exercises ANN insertion order
    // robustness.
    for (let d = 0; d < docsPerTopic; d++) {
        for (let t = 0; t < topics; t++) {
            const isDup = d < dupCount;
            const v = new Float32Array(dim);
            // Topic-axis signal
            v[t] = isDup ? 1.0 : 0.7;
            // For diverse docs, add a secondary-axis signal so they spread.
            if (!isDup) {
                // Pick a secondary axis deterministically by (topic, d)
                const secondaryAxis = ((t + d + 1) * 7) % dim;
                if (secondaryAxis !== t) {
                    v[secondaryAxis] = 0.5;
                }
            }
            // Add deterministic noise to every dim
            for (let i = 0; i < dim; i++) {
                v[i] += (rng() - 0.5) * 2 * noiseLevel;
            }
            const normalized = l2Normalize(v);
            const id = `t${t}-d${d}`;
            entries.push({ id, topic: t, isDuplicate: isDup, vector: normalized });
            relevantByTopic[t].add(id);
        }
    }
    return {
        entries,
        relevantByTopic,
        topicQueryVectors,
        options: { topics, docsPerTopic, dim, duplicateRatio, noiseLevel, seed },
    };
}
//# sourceMappingURL=synthetic-corpus.js.map