@drip_sync/drip/dist/persister/noopy_change_stream.js

Scalable incremental sync for MongoDB aggregation pipelines

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.noopyCS = noopyCS;
const mongodb_1 = require("mongodb");
const mongodb_resumetoken_decoder_1 = require("mongodb-resumetoken-decoder");
const zod_1 = __importDefault(require("zod"));
async function* noopyCS(cs) {
    try {
        let noop;
        let lastRTData;
        // We are interested in recording noops, but the change
        // stream filters them out. So we listen explicitly
        // for resumeTokenChanged.
        cs.on("resumeTokenChanged", (resumeToken) => {
            const newResumeTokenData = zod_1.default
                .object({ _data: zod_1.default.string() })
                .parse(resumeToken)._data;
            // The MongoDB Node driver emits this event
            // after each empty batch, even if the
            // resume token did not actually change.
            // So we do our own deduplication.
            if (newResumeTokenData !== lastRTData) {
                lastRTData = newResumeTokenData;
                const decoded = (0, mongodb_resumetoken_decoder_1.decodeResumeToken)(newResumeTokenData);
                noop = {
                    type: "noop",
                    rt: resumeToken,
                    // mongodb-resumetoken-decoder and the actual driver use
                    // incompatible bson versions, so translate between
                    // the two
                    ct: mongodb_1.Timestamp.fromBits(decoded.timestamp.low, decoded.timestamp.high),
                };
            }
        });
        while (true) {
            // Use tryNext instead of next to make the
            // query return if there are still no events
            // after maxAwaitTimeMS.
            // We use this to avoid getting stuck on a
            // single next(), as we do want to yield
            // periodically.
            const ce = await cs.tryNext();
            if (ce) {
                yield { type: "change", change: ce };
            }
            else if (noop) {
                // If there is no change event, but nevertheless
                // a more recent resume token, emit it as a noop
                yield noop;
            }
            else {
                // This ensures we yield once per tryNext() call
                yield { type: "nothing" };
            }
            noop = undefined;
        }
    }
    finally {
        await cs.close();
    }
}