@drip_sync/drip/dist/persister/persister.js

Scalable incremental sync for MongoDB aggregation pipelines

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.runPersister = runPersister;
const mongodb_1 = require("mongodb");
const derive_pcs_coll_name_1 = require("../cea/derive_pcs_coll_name");
const change_event_to_pcs_event_1 = require("./change_event_to_pcs_event");
const metadata_1 = require("../cea/metadata");
const promise_train_1 = require("./promise_train");
const flush_buffer_1 = require("./flush_buffer");
const noopy_change_stream_1 = require("./noopy_change_stream");
async function* runPersister(metadataClient, metadataDbName, watchCollection, options) {
    const collectionName = watchCollection.collectionName;
    const promiseTrain = new promise_train_1.PromiseTrain();
    const metadataDb = metadataClient.db(metadataDbName);
    const pcsColl = metadataDb.collection((0, derive_pcs_coll_name_1.derivePCSCollName)(collectionName));
    const metadataColl = metadataDb.collection(metadata_1.MetadataCollectionName);
    async function pushPCSEventsUpdateMetadata(events) {
        if (events.length > 0) {
            await promiseTrain.push(() => metadataClient.withSession((session) => session.withTransaction(async (session) => {
                await pcsColl.insertMany(events.map((e) => e[1]), {
                    ordered: true,
                    session,
                });
                await metadataColl.findOneAndUpdate({ _id: collectionName }, { $set: { resumeToken: events[events.length - 1][0] } }, {
                    upsert: true,
                    session,
                });
            }, {
                readConcern: mongodb_1.ReadConcernLevel.majority,
                writeConcern: { w: "majority" },
            })));
        }
    }
    const persistedResumeToken = (await metadataColl
        .find({
        _id: collectionName,
    }, { readConcern: mongodb_1.ReadConcernLevel.majority })
        .project({ _id: 0, resumeToken: 1 })
        .map((o) => metadata_1.zodDripMetadata.pick({ resumeToken: true }).parse(o))
        .toArray())[0]?.resumeToken;
    const MAX_BUFFER_LENGTH = 1000;
    const flushBuffer = new flush_buffer_1.FlushBuffer(MAX_BUFFER_LENGTH, (events) => pushPCSEventsUpdateMetadata(events));
    const minNoopIntervalMS = typeof options?.minNoopIntervalMS === "number"
        ? options.minNoopIntervalMS
        : // The default MongoDB noop interval is 10 seconds,
            // so the default here should also be less than 10 seconds.
            // It should not be too small, though, as then we cannot
            // avoid the CT loop described below.
            8000;
    const cs = watchCollection.watch([
        {
            $match: {
                operationType: { $in: ["insert", "update", "replace", "delete"] },
            },
        },
    ], {
        // To get field disambiguation
        showExpandedEvents: true,
        // We also persist pre- and post-images
        fullDocument: "required",
        fullDocumentBeforeChange: "required",
        resumeAfter: persistedResumeToken,
        readConcern: mongodb_1.ReadConcernLevel.majority,
        ...(typeof options?.maxAwaitTimeMS === "number"
            ? { maxAwaitTimeMS: options.maxAwaitTimeMS }
            : {}),
    });
    const ncs = (0, noopy_change_stream_1.noopyCS)(cs);
    try {
        let lastEventClock;
        for await (const event of ncs) {
            // This lets the caller stop us gracefully
            yield;
            if (event.type === "nothing") {
                // Nothing
            }
            else if (event.type === "noop") {
                const w = new Date();
                if (!lastEventClock ||
                    // Avoid persisting every single change in the cluster time,
                    // as this leads to an infinite loop of us persisting the CT,
                    // and the CT advancing because of the new write, if we are
                    // persisting to the same cluster we are reading the CS from.
                    w.getTime() - lastEventClock.getTime() >= minNoopIntervalMS ||
                    // Means the wall clock went backwards - so be prudent
                    // and persist the noop
                    lastEventClock > w) {
                    const noop = {
                        _id: new mongodb_1.ObjectId(),
                        ct: event.ct,
                        o: "n",
                        w: w,
                    };
                    lastEventClock = w;
                    await flushBuffer.push([event.rt, noop]);
                }
            }
            else {
                event.type;
                const pcse = (0, change_event_to_pcs_event_1.changeEventToPCSEvent)(event.change);
                if (typeof pcse !== "undefined") {
                    lastEventClock = new Date();
                    await flushBuffer.push([event.change._id, pcse]);
                }
            }
        }
    }
    finally {
        // Cancel the timer from FlushBuffer
        flushBuffer.abort();
        // Wait for the existing operations on the
        // PromiseTrain
        await promiseTrain.push(() => Promise.resolve());
        await cs.close();
    }
}