@electric-sql/d2ts
Version:
D2TS is a TypeScript implementation of Differential Dataflow.
236 lines • 9.6 kB
JavaScript
import { StreamBuilder } from '../../d2.js';
import { MessageType, } from '../../types.js';
import { MultiSet } from '../../multiset.js';
import { DifferenceStreamWriter, BinaryOperator, } from '../../graph.js';
import { SQLIndex } from '../version-index.js';
import { map, concat, negate } from '../../operators/index.js';
import { SQLiteContext } from '../context.js';
export class JoinOperatorSQLite extends BinaryOperator {
#indexA;
#indexB;
#deltaA;
#deltaB;
constructor(id, inputA, inputB, output, initialFrontier, db) {
super(id, inputA, inputB, output, initialFrontier);
this.#indexA = new SQLIndex(db, `join_a_${id}`);
this.#indexB = new SQLIndex(db, `join_b_${id}`);
this.#deltaA = new SQLIndex(db, `join_delta_a_${id}`, true);
this.#deltaB = new SQLIndex(db, `join_delta_b_${id}`, true);
}
run() {
const deltaA = this.#deltaA;
const deltaB = this.#deltaB;
try {
// Process input A
for (const message of this.inputAMessages()) {
if (message.type === MessageType.DATA) {
const { version, collection } = message.data;
// Batch the inserts
const items = [];
for (const [item, multiplicity] of collection.getInner()) {
const [key, value] = item;
items.push([key, version, [value, multiplicity]]);
}
deltaA.addValues(items);
}
else if (message.type === MessageType.FRONTIER) {
const frontier = message.data;
if (!this.inputAFrontier().lessEqual(frontier)) {
throw new Error('Invalid frontier update');
}
this.setInputAFrontier(frontier);
}
}
// Process input B
for (const message of this.inputBMessages()) {
if (message.type === MessageType.DATA) {
const { version, collection } = message.data;
// Batch the inserts
const items = [];
for (const [item, multiplicity] of collection.getInner()) {
const [key, value] = item;
items.push([key, version, [value, multiplicity]]);
}
deltaB.addValues(items);
}
else if (message.type === MessageType.FRONTIER) {
const frontier = message.data;
if (!this.inputBFrontier().lessEqual(frontier)) {
throw new Error('Invalid frontier update');
}
this.setInputBFrontier(frontier);
}
}
// Process results
const results = new Map();
// Join deltaA with existing indexB and collect results
for (const [version, collection] of deltaA.join(this.#indexB)) {
const existing = results.get(version) || new MultiSet();
existing.extend(collection);
results.set(version, existing);
}
// Append deltaA to indexA
this.#indexA.append(deltaA);
// Join indexA with deltaB and collect results
for (const [version, collection] of this.#indexA.join(deltaB)) {
const existing = results.get(version) || new MultiSet();
existing.extend(collection);
results.set(version, existing);
}
// Send all results
for (const [version, collection] of results) {
this.output.sendData(version, collection);
}
// Finally append deltaB to indexB
this.#indexB.append(deltaB);
// Update frontiers
const inputFrontier = this.inputAFrontier().meet(this.inputBFrontier());
if (!this.outputFrontier.lessEqual(inputFrontier)) {
throw new Error('Invalid frontier state');
}
if (this.outputFrontier.lessThan(inputFrontier)) {
this.outputFrontier = inputFrontier;
this.output.sendFrontier(this.outputFrontier);
this.#indexA.compact(this.outputFrontier);
this.#indexB.compact(this.outputFrontier);
}
}
finally {
// Clean up temporary indexes
deltaA.truncate();
deltaB.truncate();
}
}
}
/**
* Joins two input streams
* Persists state to SQLite
*
* @param other - The other stream to join with
* @param db - Optional SQLite database (can be injected via context)
* @param type - The type of join to perform
*/
export function join(other, db, type = 'inner') {
switch (type) {
case 'inner':
return innerJoin(other, db);
case 'anti':
return antiJoin(other, db);
case 'left':
return leftJoin(other, db);
case 'right':
return rightJoin(other, db);
case 'full':
return fullJoin(other, db);
default:
throw new Error(`Join type ${type} is invalid`);
}
}
/**
* Joins two input streams
* Persists state to SQLite
*
* @param other - The other stream to join with
* @param db - Optional SQLite database (can be injected via context)
*/
export function innerJoin(other, db) {
return (stream) => {
// Get database from context if not provided explicitly
const database = db || SQLiteContext.getDb();
if (!database) {
throw new Error('SQLite database is required for join operator. ' +
'Provide it as a parameter or use withSQLite() to inject it.');
}
const output = new StreamBuilder(stream.graph, new DifferenceStreamWriter());
const operator = new JoinOperatorSQLite(stream.graph.getNextOperatorId(), stream.connectReader(), other.connectReader(), output.writer, stream.graph.frontier(), database);
stream.graph.addOperator(operator);
stream.graph.addStream(output.connectReader());
return output;
};
}
/**
* Performs an anti-join
*
* @param other - The other stream to join with
* @param db - Optional SQLite database (can be injected via context)
*/
export function antiJoin(other, db) {
return (stream) => {
// Get database from context if not provided explicitly
const database = db || SQLiteContext.getDb();
if (!database) {
throw new Error('SQLite database is required for join operator. ' +
'Provide it as a parameter or use withSQLite() to inject it.');
}
const matchedLeft = stream.pipe(innerJoin(other, database), map(([key, [valueLeft, _valueRight]]) => [key, valueLeft]));
const anti = stream.pipe(concat(matchedLeft.pipe(negate())),
// @ts-ignore TODO: fix this
map(([key, value]) => [key, [value, null]]));
return anti;
};
}
/**
* Performs a left join
*
* @param other - The other stream to join with
* @param db - Optional SQLite database (can be injected via context)
*/
export function leftJoin(other, db) {
return (stream) => {
// Get database from context if not provided explicitly
const database = db || SQLiteContext.getDb();
if (!database) {
throw new Error('SQLite database is required for join operator. ' +
'Provide it as a parameter or use withSQLite() to inject it.');
}
const left = stream;
const right = other;
const inner = left.pipe(innerJoin(right, database));
const anti = left.pipe(antiJoin(right, database));
return inner.pipe(concat(anti));
};
}
/**
* Performs a right join
*
* @param other - The other stream to join with
* @param db - Optional SQLite database (can be injected via context)
*/
export function rightJoin(other, db) {
return (stream) => {
// Get database from context if not provided explicitly
const database = db || SQLiteContext.getDb();
if (!database) {
throw new Error('SQLite database is required for join operator. ' +
'Provide it as a parameter or use withSQLite() to inject it.');
}
const left = stream;
const right = other;
const inner = left.pipe(innerJoin(right, database));
const anti = right.pipe(antiJoin(left, database), map(([key, [a, b]]) => [key, [b, a]]));
return inner.pipe(concat(anti));
};
}
/**
* Performs a full outer join
*
* @param other - The other stream to join with
* @param db - Optional SQLite database (can be injected via context)
*/
export function fullJoin(other, db) {
return (stream) => {
// Get database from context if not provided explicitly
const database = db || SQLiteContext.getDb();
if (!database) {
throw new Error('SQLite database is required for join operator. ' +
'Provide it as a parameter or use withSQLite() to inject it.');
}
const left = stream;
const right = other;
const inner = left.pipe(innerJoin(right, database));
const antiLeft = left.pipe(antiJoin(right, database));
const antiRight = right.pipe(antiJoin(left, database), map(([key, [a, b]]) => [key, [b, a]]));
return inner.pipe(concat(antiLeft), concat(antiRight));
};
}
//# sourceMappingURL=join.js.map