@tanstack/db-ivm
Version:
Incremental View Maintenance for TanStack DB based on Differential Dataflow
202 lines (201 loc) • 8.3 kB
JavaScript
"use strict";
var __typeError = (msg) => {
throw TypeError(msg);
};
var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
var __privateMethod = (obj, member, method) => (__accessCheck(obj, member, "access private method"), method);
var _sortedValues, _comparator, _topKStart, _topKEnd, _TopKArray_instances, findIndex_fn, _index, _topK;
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
const fractionalIndexing = require("fractional-indexing");
const graph = require("../graph.cjs");
const d2 = require("../d2.cjs");
const multiset = require("../multiset.cjs");
const indexes = require("../indexes.cjs");
const utils = require("../utils.cjs");
class TopKArray {
constructor(offset, limit, comparator) {
__privateAdd(this, _TopKArray_instances);
__privateAdd(this, _sortedValues, []);
__privateAdd(this, _comparator);
__privateAdd(this, _topKStart);
__privateAdd(this, _topKEnd);
__privateSet(this, _topKStart, offset);
__privateSet(this, _topKEnd, offset + limit);
__privateSet(this, _comparator, comparator);
}
insert(value) {
const result = { moveIn: null, moveOut: null };
const index = __privateMethod(this, _TopKArray_instances, findIndex_fn).call(this, value);
const indexBefore = index === 0 ? null : getIndex(__privateGet(this, _sortedValues)[index - 1]);
const indexAfter = index === __privateGet(this, _sortedValues).length ? null : getIndex(__privateGet(this, _sortedValues)[index]);
const fractionalIndex = fractionalIndexing.generateKeyBetween(indexBefore, indexAfter);
const val = indexedValue(value, fractionalIndex);
__privateGet(this, _sortedValues).splice(index, 0, val);
if (index < __privateGet(this, _topKEnd)) {
const moveInIndex = Math.max(index, __privateGet(this, _topKStart));
if (moveInIndex < __privateGet(this, _sortedValues).length) {
result.moveIn = __privateGet(this, _sortedValues)[moveInIndex];
if (__privateGet(this, _topKEnd) < __privateGet(this, _sortedValues).length) {
result.moveOut = __privateGet(this, _sortedValues)[__privateGet(this, _topKEnd)];
}
}
}
return result;
}
/**
* Deletes a value that may or may not be in the topK.
* IMPORTANT: this assumes that the value is present in the collection
* if it's not the case it will remove the element
* that is on the position where the provided `value` would be.
*/
delete(value) {
const result = { moveIn: null, moveOut: null };
const index = __privateMethod(this, _TopKArray_instances, findIndex_fn).call(this, value);
const [removedElem] = __privateGet(this, _sortedValues).splice(index, 1);
if (index < __privateGet(this, _topKEnd)) {
result.moveOut = removedElem;
if (index < __privateGet(this, _topKStart)) {
const moveOutIndex = __privateGet(this, _topKStart) - 1;
if (moveOutIndex < __privateGet(this, _sortedValues).length) {
result.moveOut = __privateGet(this, _sortedValues)[moveOutIndex];
} else {
result.moveOut = null;
}
}
const moveInIndex = __privateGet(this, _topKEnd) - 1;
if (moveInIndex < __privateGet(this, _sortedValues).length) {
result.moveIn = __privateGet(this, _sortedValues)[moveInIndex];
}
}
return result;
}
}
_sortedValues = new WeakMap();
_comparator = new WeakMap();
_topKStart = new WeakMap();
_topKEnd = new WeakMap();
_TopKArray_instances = new WeakSet();
// TODO: see if there is a way to refactor the code for insert and delete in the topK above
// because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right
// so i have the feeling there is a common pattern here and we can implement both cases using that pattern
findIndex_fn = function(value) {
return utils.binarySearch(
__privateGet(this, _sortedValues),
indexedValue(value, ``),
(a, b) => __privateGet(this, _comparator).call(this, getValue(a), getValue(b))
);
};
class TopKWithFractionalIndexOperator extends graph.UnaryOperator {
constructor(id, inputA, output, comparator, options) {
super(id, inputA, output);
__privateAdd(this, _index, new indexes.Index());
/**
* topK data structure that supports insertions and deletions
* and returns changes to the topK.
*/
__privateAdd(this, _topK);
const limit = options.limit ?? Infinity;
const offset = options.offset ?? 0;
const compareTaggedValues = (a, b) => {
const valueComparison = comparator(untagValue(a), untagValue(b));
if (valueComparison !== 0) {
return valueComparison;
}
const tieBreakerA = getTag(a);
const tieBreakerB = getTag(b);
return tieBreakerA - tieBreakerB;
};
__privateSet(this, _topK, this.createTopK(offset, limit, compareTaggedValues));
}
createTopK(offset, limit, comparator) {
return new TopKArray(offset, limit, comparator);
}
run() {
const result = [];
for (const message of this.inputMessages()) {
for (const [item, multiplicity] of message.getInner()) {
const [key, value] = item;
this.processElement(key, value, multiplicity, result);
}
}
if (result.length > 0) {
this.output.sendData(new multiset.MultiSet(result));
}
}
processElement(key, value, multiplicity, result) {
const oldMultiplicity = __privateGet(this, _index).getMultiplicity(key, value);
__privateGet(this, _index).addValue(key, [value, multiplicity]);
const newMultiplicity = __privateGet(this, _index).getMultiplicity(key, value);
let res = {
moveIn: null,
moveOut: null
};
if (oldMultiplicity <= 0 && newMultiplicity > 0) {
const taggedValue = tagValue(value);
res = __privateGet(this, _topK).insert(taggedValue);
} else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
const taggedValue = tagValue(value);
res = __privateGet(this, _topK).delete(taggedValue);
} else ;
if (res.moveIn) {
const valueWithoutTieBreaker = mapValue(res.moveIn, untagValue);
result.push([[key, valueWithoutTieBreaker], 1]);
}
if (res.moveOut) {
const valueWithoutTieBreaker = mapValue(res.moveOut, untagValue);
result.push([[key, valueWithoutTieBreaker], -1]);
}
return;
}
}
_index = new WeakMap();
_topK = new WeakMap();
function topKWithFractionalIndex(comparator, options) {
const opts = options || {};
return (stream) => {
const output = new d2.StreamBuilder(
stream.graph,
new graph.DifferenceStreamWriter()
);
const operator = new TopKWithFractionalIndexOperator(
stream.graph.getNextOperatorId(),
stream.connectReader(),
output.writer,
comparator,
opts
);
stream.graph.addOperator(operator);
stream.graph.addStream(output.connectReader());
return output;
};
}
function indexedValue(value, index) {
return [value, index];
}
function getValue(indexedVal) {
return indexedVal[0];
}
function getIndex(indexedVal) {
return indexedVal[1];
}
function mapValue(indexedVal, f) {
return [f(getValue(indexedVal)), getIndex(indexedVal)];
}
function tagValue(value) {
return [value, utils.globalObjectIdGenerator.getId(value)];
}
function untagValue(tieBreakerTaggedValue) {
return tieBreakerTaggedValue[0];
}
function getTag(tieBreakerTaggedValue) {
return tieBreakerTaggedValue[1];
}
exports.TopKWithFractionalIndexOperator = TopKWithFractionalIndexOperator;
exports.getIndex = getIndex;
exports.getValue = getValue;
exports.indexedValue = indexedValue;
exports.topKWithFractionalIndex = topKWithFractionalIndex;
//# sourceMappingURL=topKWithFractionalIndex.cjs.map