@athenox/talos/dist/embed.js

Talos: a backend you describe instead of write. Declare a schema, write the rules in plain language, map URLs to sentences; a small built-in model turns them into query plans.

import { existsSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
let loaded;
function dirOf() {
    // Bundled, `import.meta.url` may name no file: then only the other places are looked at.
    let here;
    try {
        here = dirname(fileURLToPath(import.meta.url));
    }
    catch { }
    return [process.env.TALOS_EMBEDDER_DIR, here && join(here, '..', 'model', 'embedder'),
        join(process.cwd(), 'node_modules', '@athenox', 'talos', 'model', 'embedder'), join(process.cwd(), 'node_modules', 'talosjs', 'model', 'embedder'), join(process.cwd(), '..', 'talos', 'model', 'embedder')].find(d => d && existsSync(join(d, 'rows.i8')));
}
function load() {
    if (loaded !== undefined)
        return loaded;
    const dir = dirOf();
    if (!dir)
        return (loaded = null);
    const cfg = JSON.parse(readFileSync(join(dir, 'config.json'), 'utf8'));
    const r = readFileSync(join(dir, 'rows.i8')), s = readFileSync(join(dir, 'scales.f32'));
    loaded = {
        rows: new Int8Array(r.buffer, r.byteOffset, r.byteLength),
        scales: new Float32Array(s.buffer.slice(s.byteOffset, s.byteOffset + s.byteLength)),
        buckets: cfg.buckets, dim: cfg.dim, maxIds: cfg.maxIds,
    };
    return loaded;
}
/** whether the embedder's weights are installed */
export const hasEmbedder = () => !!load();
const fnv = (s) => {
    let h = 0x811C9DC5;
    for (let i = 0; i < s.length; i++) {
        h ^= s.charCodeAt(i);
        h = Math.imul(h, 0x01000193) >>> 0;
    }
    return h >>> 0;
};
const WORD = /[a-z0-9]+/g;
const normalize = (s) => s.toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '');
function ids(text, m) {
    const out = [];
    for (const w of normalize(text).match(WORD) ?? []) {
        out.push(fnv(`w:${w}`) % m.buckets);
        const g = `<${w}>`;
        for (const n of [3, 4, 5])
            for (let i = 0; i + n <= g.length; i++)
                out.push(fnv(g.slice(i, i + n)) % m.buckets);
        if (out.length >= m.maxIds)
            break;
    }
    return out.length ? out.slice(0, m.maxIds) : [0];
}
// ponytail: an unbounded map keyed by text; bound it (LRU) if texts are endless.
const cache = new Map();
/** A text's vector, unit length; null when the embedder is not installed. */
export function embedText(text) {
    const m = load();
    if (!m)
        return null;
    const hit = cache.get(text);
    if (hit)
        return hit;
    const v = new Float32Array(m.dim);
    const list = ids(text, m);
    for (const id of list) {
        const s = m.scales[id] / list.length, base = id * m.dim;
        for (let d = 0; d < m.dim; d++)
            v[d] += m.rows[base + d] * s;
    }
    let n = 0;
    for (const x of v)
        n += x * x;
    n = Math.sqrt(n) || 1;
    for (let d = 0; d < m.dim; d++)
        v[d] /= n;
    cache.set(text, v);
    return v;
}
export const cosine = (a, b) => {
    let s = 0;
    for (let i = 0; i < a.length; i++)
        s += a[i] * b[i];
    return s;
};
/**
 * The labels a text is closest to, best first: tagging an article with the
 * app's categories, routing a message to a department.
 */
export function classify(text, labels, top = 1) {
    const v = embedText(text);
    if (!v)
        return [];
    return Object.entries(labels)
        .map(([label, describe]) => ({ label, score: cosine(v, embedText(describe)) }))
        .sort((a, b) => b.score - a.score).slice(0, top);
}