@athenox/talos/dist/recommend.js

Talos: a backend you describe instead of write. Declare a schema, write the rules in plain language, map URLs to sentences; a small built-in model turns them into query plans.

import { cosine, embedText, hasEmbedder } from './embed.js';
/** the embedder's share in text likeness (measured best between 0.2 and 0.3, see talos/test/embed.ts) */
const MEANING = 0.3;
/**
 * Rows ordered by how close their text is to a row or a text: "à lire aussi",
 * near-duplicates. Uses the embedder, and word overlap when it is absent.
 */
export function similar(rows, o) {
    const textOf = (r) => o.text.map(c => String(r[c] ?? '')).join('. ');
    const pool = rows.filter(r => r.id !== o.exclude);
    // TF-IDF over the rows, mixed with the embedder's meaning as in `rank`.
    const df = new Map();
    const bags = pool.map(r => new Set(words(textOf(r))));
    for (const b of bags)
        for (const w of b)
            df.set(w, (df.get(w) ?? 0) + 1);
    const vec = (t) => {
        const tf = new Map();
        for (const w of words(t))
            tf.set(w, (tf.get(w) ?? 0) + 1);
        const v = new Map();
        let n = 0;
        for (const [w, c] of tf) {
            const x = (1 + Math.log(c)) * (Math.log((pool.length + 1) / ((df.get(w) ?? 0) + 1)) + 1);
            v.set(w, x);
            n += x * x;
        }
        n = Math.sqrt(n) || 1;
        for (const [w, x] of v)
            v.set(w, x / n);
        return v;
    };
    const tv = vec(o.to), target = embedText(o.to);
    return pool
        .map(r => {
        const rv = vec(textOf(r));
        let w = 0;
        for (const [t, x] of tv)
            w += x * (rv.get(t) ?? 0);
        const e = target && embedText(textOf(r));
        const s = e ? (1 - MEANING) * w + MEANING * cosine(target, e) : w;
        return { ...r, score: Math.round(s * 1000) / 1000 };
    })
        .filter(r => r.score >= (o.min ?? -1))
        .sort((a, b) => b.score - a.score).slice(0, o.limit ?? 5);
}
const WORD = /[a-z0-9]{3,}/g;
const STOP = new Set(('les des une dans pour avec sur par est sont qui que quoi aux ces cette son ses leur leurs plus moins '
    + 'tout tous toute toutes the and for with are was were this that from have has not but you your our').split(' '));
const words = (s) => (String(s ?? '').toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '').match(WORD) ?? [])
    .filter(w => !STOP.has(w));
export async function rank(rows, o, store, q) {
    const w = { content: 1, together: 1, popular: 0.3, recent: 0.3, ...o.weights };
    const ids = rows.map(r => r.id);
    const it = o.interactions;
    // The reader's items, and who else touched the candidates.
    const mine = new Set(it && o.user != null
        ? (await store.all(`SELECT DISTINCT ${q(it.item)} AS i FROM ${q(it.table)} WHERE ${q(it.user)} = ?`, o.user)).map((r) => r.i)
        : []);
    const counts = new Map();
    const together = new Map();
    if (it && ids.length) {
        const marks = ids.map(() => '?').join(', ');
        for (const r of await store.all(`SELECT ${q(it.item)} AS i, COUNT(*) AS n FROM ${q(it.table)} WHERE ${q(it.item)} IN (${marks}) GROUP BY ${q(it.item)}`, ...ids))
            counts.set(r.i, Number(r.n));
        if (mine.size) {
            const seen = [...mine];
            // Readers who share an item with this reader, weighted by how many they share.
            const peers = await store.all(`SELECT ${q(it.user)} AS u, COUNT(*) AS n FROM ${q(it.table)} WHERE ${q(it.item)} IN (${seen.map(() => '?').join(', ')})`
                + ` AND ${q(it.user)} <> ? GROUP BY ${q(it.user)}`, ...seen, o.user);
            if (peers.length) {
                const byPeer = new Map(peers.map((p) => [p.u, Number(p.n)]));
                const hits = await store.all(`SELECT ${q(it.user)} AS u, ${q(it.item)} AS i FROM ${q(it.table)} WHERE ${q(it.user)} IN (${peers.map(() => '?').join(', ')})`
                    + ` AND ${q(it.item)} IN (${marks})`, ...peers.map((p) => p.u), ...ids);
                const once = new Set();
                for (const h of hits) {
                    const key = `${h.u}:${h.i}`;
                    if (once.has(key))
                        continue;
                    once.add(key);
                    together.set(h.i, (together.get(h.i) ?? 0) + (byPeer.get(h.u) ?? 0));
                }
            }
        }
    }
    // TF-IDF over the candidates' text, the profile being the reader's items among them.
    const docs = new Map();
    const df = new Map();
    if (o.text?.length)
        for (const r of rows) {
            const tf = new Map();
            for (const t of o.text.flatMap(c => words(r[c])))
                tf.set(t, (tf.get(t) ?? 0) + 1);
            docs.set(r.id, tf);
            for (const t of tf.keys())
                df.set(t, (df.get(t) ?? 0) + 1);
        }
    const idf = (t) => Math.log((rows.length + 1) / ((df.get(t) ?? 0) + 1)) + 1;
    const vec = (tf) => {
        const v = new Map();
        let n = 0;
        for (const [t, c] of tf) {
            const x = (1 + Math.log(c)) * idf(t);
            v.set(t, x);
            n += x * x;
        }
        n = Math.sqrt(n) || 1;
        for (const [t, x] of v)
            v.set(t, x / n);
        return v;
    };
    const vecs = new Map([...docs].map(([id, tf]) => [id, vec(tf)]));
    const profile = new Map();
    for (const id of mine)
        for (const [t, x] of vecs.get(id) ?? [])
            profile.set(t, (profile.get(t) ?? 0) + x);
    const pn = Math.sqrt([...profile.values()].reduce((s, x) => s + x * x, 0)) || 1;
    // And by meaning, when the embedder is installed: the mean of what the
    // reader read, against each candidate — articles on the same subject that
    // share no word still meet here.
    const meaning = new Map();
    if (o.text?.length && hasEmbedder())
        for (const r of rows)
            meaning.set(r.id, embedText(o.text.map(c => String(r[c] ?? '')).join('. ')));
    let taste = null;
    for (const id of mine) {
        const v = meaning.get(id);
        if (!v)
            continue;
        taste ??= new Float32Array(v.length);
        for (let d = 0; d < v.length; d++)
            taste[d] += v[d];
    }
    const content = (id) => {
        const v = vecs.get(id);
        let words = 0;
        if (v && profile.size) {
            for (const [t, x] of v)
                words += x * (profile.get(t) ?? 0);
            words /= pn;
        }
        const m = meaning.get(id);
        if (!taste || !m)
            return words;
        let tn = 0;
        for (const x of taste)
            tn += x * x;
        // 70 % words, 30 % meaning: on held-out Wikipedia passages words alone
        // found the right one 37 % of the time, meaning alone 27 %, this mix 40 %.
        return (1 - MEANING) * words + MEANING * Math.max(0, cosine(m, taste) / (Math.sqrt(tn) || 1));
    };
    const maxCount = Math.max(1, ...counts.values());
    const maxTogether = Math.max(1, ...together.values());
    const halfLife = (o.halfLife ?? 30) * 86_400_000;
    const now = Date.now();
    const recent = (r) => {
        if (!o.date || !r[o.date])
            return 0;
        const t = new Date(r[o.date]).getTime();
        return Number.isNaN(t) ? 0 : Math.pow(0.5, Math.max(0, now - t) / halfLife);
    };
    return rows
        .filter(r => o.keepSeen || !mine.has(r.id))
        .map(r => ({ r, s: w.content * content(r.id) + w.together * (together.get(r.id) ?? 0) / maxTogether
            + w.popular * (counts.get(r.id) ?? 0) / maxCount + w.recent * recent(r) }))
        .sort((a, b) => b.s - a.s)
        .slice(0, o.limit ?? 10)
        .map(({ r, s }) => ({ ...r, score: Math.round(s * 1000) / 1000 }));
}