ndx
Version:
Lightweight Full-Text Indexing and Searching Library.
229 lines • 6.24 kB
JavaScript
const _Int32Array = Int32Array;
const _Float64Array = Float64Array;
const _Map = Map;
const SEARCH_CONTEXT = Object.seal({
found: false,
i: 0,
});
const findByCharCode = (array, charCode) => {
const ctx = SEARCH_CONTEXT;
let low = 0;
let high = array.length;
while (low < high) {
const mid = (low + high) >> 1;
const c = array[mid].k - charCode;
if (c < 0) {
low = mid + 1;
}
else if (c > 0) {
high = mid;
}
else {
ctx.found = true;
ctx.i = mid;
return;
}
}
ctx.found = false;
ctx.i = low;
};
/**
* Creates an Index.
*
* @typeparam T Document key.
* @param fieldsNum Number of fields.
* @returns {@link Index}
*/
export const createIndex = (fieldsNum, tokenizer, filter) => ({
docs: new _Map(),
root: createInvertedIndexNode(0),
fSum: new _Float64Array(fieldsNum),
fAvg: new _Float64Array(fieldsNum),
removed: 0,
tokenizer,
filter,
});
/**
* Creates inverted index node.
*
* @typeparam T Document key.
* @param k Char code.
* @returnd {@link InvertedIndexNode} instance.
*/
const createInvertedIndexNode = (k) => ({
k,
c: null,
d: null,
});
/**
* Finds inverted index node that matches the `term`.
*
* @typeparam T Document key.
* @param node Root node.
* @param term Term.
* @returns Inverted index node that contains `term` or an `undefined` value.
*/
export const findInvertedIndexNode = (node, term) => {
const ctx = SEARCH_CONTEXT;
let i = 0;
while (node !== void 0 && i < term.length) {
const c = node.c;
if (c === null) {
return void 0;
}
findByCharCode(c, term.charCodeAt(i++));
if (ctx.found === false) {
return void 0;
}
node = c[ctx.i];
}
return node;
};
/**
* Adds a document to the index.
*
* @typeparam T Document key.
* @typeparam D Document type.
* @param index {@link DocumentIndex}.
* @param fieldGetters Field getters.
* @param key Document key.
* @param doc Document.
*/
export const indexAdd = (index, fieldGetters, key, doc) => {
const { root, fSum, fAvg, docs, tokenizer, filter } = index;
const termCounts = new _Map();
const fCount = new _Int32Array(fieldGetters.length);
for (let i = 0; i < fieldGetters.length; i++) {
const field = fieldGetters[i](doc);
if (field !== void 0) {
// tokenize text
const terms = tokenizer(field);
// filter and count terms, ignore empty strings
let filteredTermsCount = 0;
for (let j = 0; j < terms.length; j++) {
const term = filter(terms[j]);
if (term !== "") {
filteredTermsCount++;
let fieldTermCounts = termCounts.get(term);
if (fieldTermCounts === void 0) {
fieldTermCounts = new _Int32Array(fSum.length);
termCounts.set(term, fieldTermCounts);
}
fieldTermCounts[i] += 1;
}
}
fSum[i] += filteredTermsCount;
fAvg[i] = fSum[i] / (docs.size + 1);
fCount[i] = filteredTermsCount;
}
}
const details = { key, fCount, removed: false };
docs.set(key, details);
termCounts.forEach((termFrequency, term) => {
const ctx = SEARCH_CONTEXT;
let node = root;
for (let i = 0; i < term.length; i++) {
const charCode = term.charCodeAt(i);
let newNode;
if (node.c === null) {
newNode = createInvertedIndexNode(charCode);
node.c = [newNode];
node = newNode;
continue;
}
findByCharCode(node.c, charCode);
if (ctx.found === false) {
newNode = createInvertedIndexNode(charCode);
node.c.splice(ctx.i, 0, newNode);
node = newNode;
}
else {
node = node.c[ctx.i];
}
}
const doc = { details, tf: termFrequency };
if (node.d === null) {
node.d = [doc];
}
else {
node.d.push(doc);
}
});
};
/**
* Remove document from the index.
*
* @typeparam T Document key.
* @param index {@link DocumentIndex}.
* @param key Document key.
*/
export const indexRemove = (index, key) => {
const { docs, fSum, fAvg } = index;
const docDetails = docs.get(key);
if (docDetails !== void 0) {
index.removed++;
docDetails.removed = true;
docs.delete(key);
for (let i = 0; i < fSum.length; i++) {
const fieldLength = docDetails.fCount[i];
if (fieldLength > 0) {
fSum[i] -= fieldLength;
fAvg[i] = fSum[i] / docs.size;
}
}
}
};
/**
* Recursively cleans up removed documents from the index.
*
* @typeparam T Document key.
* @param node {@link InvertedIndexNode}
* @returns `1` when subtree contains any document.
*/
function _vacuumIndex(node) {
let i = 0;
let ret = 0;
const d = node.d;
const c = node.c;
if (d !== null) {
while (i < d.length) {
const doc = d[i];
if (doc.details.removed === true) {
if (d.length > 1) {
d[i] = d[d.length - 1];
}
d.pop();
continue;
}
i++;
}
if (d.length > 0) {
ret = 1;
}
}
if (c !== null) {
i = 0;
while (i < c.length) {
const r = _vacuumIndex(c[i]);
ret |= r;
if (r === 0) {
c.splice(i, 1);
}
else {
i++;
}
}
}
return ret;
}
/**
* Cleans up removed documents from the {@link DocumentIndex}.
*
* @typeparam T Document key.
* @param index {@link DocumentIndex}.
*/
export function indexVacuum(index) {
_vacuumIndex(index.root);
index.removed = 0;
}
//# sourceMappingURL=index.js.map