@electric-sql/d2ts
Version:
D2TS is a TypeScript implementation of Differential Dataflow.
113 lines • 4.95 kB
JavaScript
import { MessageType } from '../types.js';
import { DifferenceStreamWriter, UnaryOperator, } from '../graph.js';
import { StreamBuilder } from '../d2.js';
import { MultiSet } from '../multiset.js';
import { Index } from '../version-index.js';
/**
* Base operator for reduction operations
*/
export class ReduceOperator extends UnaryOperator {
#index = new Index();
#indexOut = new Index();
#keysTodo = new Map();
#f;
constructor(id, inputA, output, f, initialFrontier) {
super(id, inputA, output, initialFrontier);
this.#f = f;
}
run() {
for (const message of this.inputMessages()) {
if (message.type === MessageType.DATA) {
const { version, collection } = message.data;
for (const [item, multiplicity] of collection.getInner()) {
const [key, value] = item;
this.#index.addValue(key, version, [value, multiplicity]);
let todoSet = this.#keysTodo.get(version);
if (!todoSet) {
todoSet = new Set();
this.#keysTodo.set(version, todoSet);
}
todoSet.add(key);
// Add key to all join versions
for (const v2 of this.#index.versions(key)) {
const joinVersion = version.join(v2);
let joinTodoSet = this.#keysTodo.get(joinVersion);
if (!joinTodoSet) {
joinTodoSet = new Set();
this.#keysTodo.set(joinVersion, joinTodoSet);
}
joinTodoSet.add(key);
}
}
}
else if (message.type === MessageType.FRONTIER) {
const frontier = message.data;
if (!this.inputFrontier().lessEqual(frontier)) {
throw new Error('Invalid frontier update');
}
this.setInputFrontier(frontier);
}
}
// Find versions that are complete
const finishedVersions = Array.from(this.#keysTodo.entries())
.filter(([version]) => !this.inputFrontier().lessEqualVersion(version))
.sort(([a], [b]) => {
return a.lessEqual(b) ? -1 : 1;
});
for (const [version, keys] of finishedVersions) {
const result = [];
for (const key of keys) {
const curr = this.#index.reconstructAt(key, version);
const currOut = this.#indexOut.reconstructAt(key, version);
const out = this.#f(curr);
// Calculate delta between current and previous output
const delta = new Map();
const values = new Map();
for (const [value, multiplicity] of out) {
const valueKey = JSON.stringify(value);
values.set(valueKey, value);
delta.set(valueKey, (delta.get(valueKey) || 0) + multiplicity);
}
for (const [value, multiplicity] of currOut) {
const valueKey = JSON.stringify(value);
values.set(valueKey, value);
delta.set(valueKey, (delta.get(valueKey) || 0) - multiplicity);
}
// Add non-zero deltas to result
for (const [valueKey, multiplicity] of delta) {
const value = values.get(valueKey);
if (multiplicity !== 0) {
result.push([[key, value], multiplicity]);
this.#indexOut.addValue(key, version, [value, multiplicity]);
}
}
}
if (result.length > 0) {
this.output.sendData(version, new MultiSet(result));
}
this.#keysTodo.delete(version);
}
if (!this.outputFrontier.lessEqual(this.inputFrontier())) {
throw new Error('Invalid frontier state');
}
if (this.outputFrontier.lessThan(this.inputFrontier())) {
this.outputFrontier = this.inputFrontier();
this.output.sendFrontier(this.outputFrontier);
this.#index.compact(this.outputFrontier);
this.#indexOut.compact(this.outputFrontier);
}
}
}
/**
* Reduces the elements in the stream by key
*/
export function reduce(f) {
return (stream) => {
const output = new StreamBuilder(stream.graph, new DifferenceStreamWriter());
const operator = new ReduceOperator(stream.graph.getNextOperatorId(), stream.connectReader(), output.writer, f, stream.graph.frontier());
stream.graph.addOperator(operator);
stream.graph.addStream(output.connectReader());
return output;
};
}
//# sourceMappingURL=reduce.js.map