@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
205 lines (204 loc) • 7.6 kB
JavaScript
/**
* In-memory knowledge index construction.
*
* Index construction is an application-process lifecycle operation, not a
* per-session one. A complete `KnowledgeIndexSnapshot` is built once from the
* normalized entries and then queried by every turn.
*
* The lexical scorer is a compact field-aware BM25 (BM25F-style): each of the
* five text fields is scored independently and combined with per-field
* weights, retaining the per-field breakdown so a retrieval decision is
* explainable in traces. No embedding model or vector store is involved.
*/
import { normalizePhrases, normalizeText, tokenize } from "./normalize.js";
const FIELD_NAMES = [
"title",
"aliases",
"keywords",
"summary",
"body",
];
/** Okapi BM25 saturation and length-normalization parameters. */
const BM25_K1 = 1.5;
const BM25_B = 0.75;
/**
* Field-aware BM25 over the document set. Query text is scored per field and
* the weighted per-field scores are summed. Satisfies the structural
* `KnowledgeLexicalSearcher` type held by a snapshot.
*/
export class KnowledgeLexicalIndex {
weights;
// field -> (term -> (docId -> term frequency))
postings = new Map();
// field -> (docId -> field length in tokens)
fieldLengths = new Map();
// field -> total tokens across all docs (for average length)
fieldTokenTotals = new Map();
// field -> average field length (computed in finalize)
avgFieldLength = new Map();
docCount = 0;
constructor(weights) {
this.weights = weights;
for (const field of FIELD_NAMES) {
this.postings.set(field, new Map());
this.fieldLengths.set(field, new Map());
this.fieldTokenTotals.set(field, 0);
}
}
add(document) {
this.docCount += 1;
for (const field of FIELD_NAMES) {
const tokens = document.fields[field];
const postingsForField = this.postings.get(field);
const lengthsForField = this.fieldLengths.get(field);
if (!postingsForField || !lengthsForField) {
continue;
}
lengthsForField.set(document.id, tokens.length);
this.fieldTokenTotals.set(field, (this.fieldTokenTotals.get(field) ?? 0) + tokens.length);
const termFrequency = new Map();
for (const token of tokens) {
termFrequency.set(token, (termFrequency.get(token) ?? 0) + 1);
}
for (const [term, frequency] of termFrequency) {
const docMap = postingsForField.get(term);
if (docMap) {
docMap.set(document.id, frequency);
}
else {
postingsForField.set(term, new Map([[document.id, frequency]]));
}
}
}
}
finalize() {
for (const field of FIELD_NAMES) {
const total = this.fieldTokenTotals.get(field) ?? 0;
this.avgFieldLength.set(field, this.docCount > 0 ? total / this.docCount : 0);
}
}
search(queryTokens, topK, eligibleEntryIds) {
if (this.docCount === 0 || queryTokens.length === 0) {
return [];
}
const uniqueTerms = Array.from(new Set(queryTokens));
const totals = new Map();
const breakdowns = new Map();
for (const field of FIELD_NAMES) {
const weight = this.weights[field];
const avgdl = this.avgFieldLength.get(field) ?? 0;
const postingsForField = this.postings.get(field);
const lengthsForField = this.fieldLengths.get(field);
if (weight <= 0 || avgdl <= 0 || !postingsForField || !lengthsForField) {
continue;
}
for (const term of uniqueTerms) {
const docMap = postingsForField.get(term);
if (!docMap) {
continue;
}
const documentFrequency = docMap.size;
const idf = Math.log(1 +
(this.docCount - documentFrequency + 0.5) /
(documentFrequency + 0.5));
for (const [docId, frequency] of docMap) {
if (eligibleEntryIds && !eligibleEntryIds.has(docId)) {
continue;
}
const length = lengthsForField.get(docId) ?? 0;
const denominator = frequency + BM25_K1 * (1 - BM25_B + BM25_B * (length / avgdl));
const contribution = ((idf * (frequency * (BM25_K1 + 1))) / denominator) * weight;
totals.set(docId, (totals.get(docId) ?? 0) + contribution);
const breakdown = breakdowns.get(docId) ?? {};
breakdown[field] = (breakdown[field] ?? 0) + contribution;
breakdowns.set(docId, breakdown);
}
}
}
const matches = [];
for (const [id, score] of totals) {
matches.push({ id, score, fieldScores: breakdowns.get(id) ?? {} });
}
matches.sort((left, right) => right.score - left.score || compareIds(left.id, right.id));
return matches.slice(0, topK);
}
}
/** Deterministic tiebreak so equal scores produce a stable order. */
const compareIds = (left, right) => {
if (left < right) {
return -1;
}
if (left > right) {
return 1;
}
return 0;
};
const addToIndex = (index, key, id) => {
const bucket = index.get(key);
if (bucket) {
bucket.add(id);
}
else {
index.set(key, new Set([id]));
}
};
/**
* Build the internal search document from a normalized entry. `exactKeys`
* carry whole-phrase identifiers (entry id and title); each
* text field is tokenized for the field-aware scorer.
*/
export const buildDocument = (entry) => {
const exactKeys = normalizePhrases([entry.id, entry.title]);
return {
id: entry.id,
exactKeys,
fields: {
title: tokenize(entry.title),
aliases: entry.aliases.flatMap((alias) => tokenize(alias)),
keywords: entry.keywords.flatMap((keyword) => tokenize(keyword)),
summary: tokenize(entry.summary),
body: tokenize(entry.body),
},
};
};
/**
* Build the complete immutable snapshot: entry map, exact/alias/relation
* indexes, and the field-aware lexical index.
*/
export const buildIndexSnapshot = (entries, weights) => {
const entriesById = new Map();
const exactIndex = new Map();
const aliasIndex = new Map();
const relationIndex = new Map();
const lexical = new KnowledgeLexicalIndex(weights);
for (const entry of entries) {
entriesById.set(entry.id, entry);
const document = buildDocument(entry);
for (const key of document.exactKeys) {
addToIndex(exactIndex, key, entry.id);
}
for (const alias of entry.aliases) {
const key = normalizeText(alias);
if (key) {
addToIndex(aliasIndex, key, entry.id);
}
}
const relations = [...entry.relatedEntryIds];
if (entry.parentEntryId) {
relations.push(entry.parentEntryId);
}
if (relations.length > 0) {
relationIndex.set(entry.id, relations);
}
lexical.add(document);
}
lexical.finalize();
return {
entriesById,
exactIndex,
aliasIndex,
relationIndex,
lexical,
entryCount: entries.length,
};
};