@electric-sql/d2ts/dist/sqlite/operators/buffer.js

D2TS is a TypeScript implementation of Differential Dataflow.

import { StreamBuilder } from '../../d2.js';
import { MessageType, } from '../../types.js';
import { MultiSet } from '../../multiset.js';
import { DifferenceStreamWriter, UnaryOperator, } from '../../graph.js';
import { Version } from '../../order.js';
import { SQLiteContext } from '../context.js';
/**
 * Operator that buffers collections at each version, persisting state to SQLite
 * Ensures that completed versions are sent to the output as a whole, and in order
 */
export class BufferOperatorSQLite extends UnaryOperator {
    #preparedStatements;
    constructor(id, inputA, output, initialFrontier, db) {
        super(id, inputA, output, initialFrontier);
        // Initialize database
        db.exec(`
      CREATE TABLE IF NOT EXISTS buffer_collections_${this.id} (
        version TEXT PRIMARY KEY,
        collection TEXT NOT NULL
      )
    `);
        db.exec(`
      CREATE INDEX IF NOT EXISTS buffer_collections_${this.id}_version
      ON buffer_collections_${this.id}(version);
    `);
        // Prepare statements
        this.#preparedStatements = {
            insert: db.prepare(`INSERT INTO buffer_collections_${this.id} (version, collection) VALUES (@version, @collection)`),
            update: db.prepare(`UPDATE buffer_collections_${this.id} SET collection = @collection WHERE version = @version`),
            get: db.prepare(`SELECT collection FROM buffer_collections_${this.id} WHERE version = ?`),
            delete: db.prepare(`DELETE FROM buffer_collections_${this.id} WHERE version = ?`),
            getAllVersions: db.prepare(`SELECT version, collection FROM buffer_collections_${this.id}`),
        };
    }
    run() {
        for (const message of this.inputMessages()) {
            if (message.type === MessageType.DATA) {
                const { version, collection } = message.data;
                // Get existing collection or create new one
                const existingData = this.#preparedStatements.get.get(version.toJSON());
                const existingCollection = existingData
                    ? MultiSet.fromJSON(existingData.collection)
                    : new MultiSet();
                // Merge collections
                existingCollection.extend(collection);
                // Store updated collection
                if (existingData) {
                    this.#preparedStatements.update.run({
                        version: version.toJSON(),
                        collection: existingCollection.toJSON(),
                    });
                }
                else {
                    this.#preparedStatements.insert.run({
                        version: version.toJSON(),
                        collection: existingCollection.toJSON(),
                    });
                }
            }
            else if (message.type === MessageType.FRONTIER) {
                const frontier = message.data;
                if (!this.inputFrontier().lessEqual(frontier)) {
                    throw new Error('Invalid frontier update');
                }
                this.setInputFrontier(frontier);
            }
        }
        // Find versions that are complete (not covered by input frontier)
        const allVersions = this.#preparedStatements.getAllVersions.all();
        const finishedVersions = allVersions
            .map((row) => ({
            version: Version.fromJSON(row.version),
            collection: MultiSet.fromJSON(row.collection),
        }))
            .filter(({ version }) => !this.inputFrontier().lessEqualVersion(version));
        // Process and remove finished versions
        for (const { version, collection } of finishedVersions) {
            this.#preparedStatements.delete.run(version.toJSON());
            this.output.sendData(version, collection);
        }
        if (!this.outputFrontier.lessEqual(this.inputFrontier())) {
            throw new Error('Invalid frontier state');
        }
        if (this.outputFrontier.lessThan(this.inputFrontier())) {
            this.outputFrontier = this.inputFrontier();
            this.output.sendFrontier(this.outputFrontier);
        }
    }
}
/**
 * Buffers the elements in the stream
 * Ensures that completed versions are sent to the output as a whole, and in order
 * Persists state to SQLite
 *
 * @param db - Optional SQLite database (can be injected via context)
 */
export function buffer(db) {
    return (stream) => {
        // Get database from context if not provided explicitly
        const database = db || SQLiteContext.getDb();
        if (!database) {
            throw new Error('SQLite database is required for buffer operator. ' +
                'Provide it as a parameter or use withSQLite() to inject it.');
        }
        const output = new StreamBuilder(stream.graph, new DifferenceStreamWriter());
        const operator = new BufferOperatorSQLite(stream.graph.getNextOperatorId(), stream.connectReader(), output.writer, stream.graph.frontier(), database);
        stream.graph.addOperator(operator);
        stream.graph.addStream(output.connectReader());
        return output;
    };
}
//# sourceMappingURL=buffer.js.map