@athenox/talos/dist/search.js

Talos: a backend you describe instead of write. Declare a schema, write the rules in plain language, map URLs to sentences; a small built-in model turns them into query plans.

import { cosine, embedText, hasEmbedder } from './embed.js';
const WORD = /[a-z0-9]+/g;
const STOP = new Set(('le la les l de du des d un une et en au aux a y ou dans pour avec sur par est sont qui que ces cette son ses '
    + 'leur plus ne pas se sa ce the of to in on at is it an a and for with are was this that from').split(' '));
const tokens = (s) => (s.toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '').match(WORD) ?? []).filter(w => w.length > 1 && !STOP.has(w));
// ponytail: unbounded, keyed by text; bound it (LRU) for very large, fast-changing tables.
const cache = new Map();
const toks = (s) => { let t = cache.get(s); if (!t) {
    t = tokens(s);
    cache.set(s, t);
} return t; };
const stripHtml = (s) => s.replace(/<[^>]*>/g, ' ').replace(/&nbsp;|&#160;/g, ' ').replace(/&[a-z]+;/g, ' ');
/** at most one edit (insert, delete, substitute) between a and b */
function oneEdit(a, b) {
    if (Math.abs(a.length - b.length) > 1)
        return false;
    let i = 0, j = 0, edits = 0;
    while (i < a.length && j < b.length) {
        if (a[i] === b[j]) {
            i++;
            j++;
            continue;
        }
        if (++edits > 1)
            return false;
        if (a.length > b.length)
            i++;
        else if (a.length < b.length)
            j++;
        else {
            i++;
            j++;
        }
    }
    return edits + (a.length - i) + (b.length - j) <= 1;
}
export function find(rows, o) {
    const query = tokens(o.q);
    if (!query.length)
        return [];
    const text = (r, f) => { const v = String(r[f] ?? ''); return o.html ? stripHtml(v) : v; };
    if (o.suggest) {
        const last = query.at(-1);
        return rows.filter(r => toks(text(r, o.suggest)).some(w => w.startsWith(last))).slice(0, o.limit ?? 5);
    }
    // Document frequencies and mean field lengths, over the candidates.
    const fields = Object.entries(o.fields);
    const docs = rows.map(r => Object.fromEntries(fields.map(([f]) => [f, toks(text(r, f))])));
    const vocab = new Map();
    for (const d of docs)
        for (const w of new Set(fields.flatMap(([f]) => d[f])))
            vocab.set(w, (vocab.get(w) ?? 0) + 1);
    const avg = Object.fromEntries(fields.map(([f]) => [f, docs.reduce((s, d) => s + d[f].length, 0) / (docs.length || 1) || 1]));
    // Which row words each query word stands for: itself, what it starts, a typo of it.
    const forms = query.map(q => [...vocab.keys()].filter(w => w === q || (q.length >= 3 && w.startsWith(q)) || (q.length >= 5 && oneEdit(q, w))));
    const idf = (w) => Math.log(1 + (rows.length - (vocab.get(w) ?? 0) + 0.5) / ((vocab.get(w) ?? 0) + 0.5));
    const k1 = 1.2, b = 0.75;
    const lexical = docs.map(d => {
        let s = 0, matched = 0;
        forms.forEach((fs, qi) => {
            let best = 0;
            for (const w of fs) {
                // An exact word counts fully; a prefix or a typo a little less.
                const exact = w === query[qi] ? 1 : 0.7;
                for (const [f, weight] of fields) {
                    const tf = d[f].filter(x => x === w).length;
                    if (!tf)
                        continue;
                    const len = d[f].length;
                    best = Math.max(best, exact * weight * idf(w) * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * len / avg[f])));
                }
            }
            if (best > 0)
                matched++;
            s += best;
        });
        return { s, matched };
    });
    const top = Math.max(1e-9, ...lexical.map(l => l.s));
    const qv = hasEmbedder() ? embedText(o.q) : null;
    return rows
        .map((r, i) => {
        const words = lexical[i].s / top;
        const meaning = qv ? Math.max(0, cosine(qv, embedText(fields.map(([f]) => text(r, f)).join('. ').slice(0, 2000)))) : 0;
        // Meaning alone must be clear to bring in a row that matched no word:
        // measured, a related text scores about 0.2 and unrelated ones about 0.
        const keep = lexical[i].matched > 0 || meaning > 0.15;
        return { r, keep, score: 0.7 * words + 0.3 * meaning };
    })
        .filter(x => x.keep)
        .sort((a, b) => b.score - a.score)
        .slice(0, o.limit ?? 20)
        .map(({ r, score }) => ({ ...r, score: Math.round(score * 1000) / 1000 }));
}