@drip_sync/drip/dist/cea/cea.js

Scalable incremental sync for MongoDB aggregation pipelines

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CEACursorTooOldError = exports.CEACursorNotFoundError = exports.CEACannotResumeError = void 0;
exports.dripCEAStart = dripCEAStart;
exports.dripCEAResume = dripCEAResume;
const mongodb_1 = require("mongodb");
const zod_1 = __importDefault(require("zod"));
const stream_algebra_1 = require("./stream_algebra");
const oid_less_1 = require("./oid_less");
const derive_pcs_coll_name_1 = require("./derive_pcs_coll_name");
const scope_stage_1 = require("./scope_ppl/scope_stage");
const invert_ppl_1 = require("./invert_ppl");
const parse_pipeline_1 = require("./parse_ppl/parse_pipeline");
const synth_pipeline_1 = require("./parse_ppl/synth_pipeline");
const strip_to_gate_1 = require("./strip_to_gate");
function pcseLT(a, b) {
    return a.ct.lt(b.ct) || (a.ct.eq(b.ct) && (0, oid_less_1.oidLT)(a._id, b._id));
}
async function* dripCEAStart(db, collectionName, ccStart, pipeline, processingPipeline, options) {
    const pcseId = zod_1.default
        .object({ _id: zod_1.default.instanceof(mongodb_1.ObjectId) })
        .optional()
        .parse((await db
        .collection((0, derive_pcs_coll_name_1.derivePCSCollName)(collectionName))
        .find({ ct: { $lt: ccStart } }, { readConcern: mongodb_1.ReadConcernLevel.majority })
        .sort({ ct: -1, _id: -1 })
        .project({ _id: 1 })
        .limit(1)
        .toArray())[0])?._id;
    if (!pcseId) {
        throw new CEACursorNotFoundError();
    }
    yield* dripCEAResume(db, collectionName, {
        id: pcseId,
    }, pipeline, processingPipeline, options);
}
async function* dripCEAResume(db, collectionName, cursor, pipeline, processingPipeline, options) {
    const pipelineParsed = (0, parse_pipeline_1.parsePipeline)(pipeline);
    const pipelineScopedToAfter = (0, synth_pipeline_1.synthPipeline)((0, scope_stage_1.scopeStages)(pipelineParsed, "a"));
    const processingPipelineScopedToAfter = processingPipeline
        ? (0, synth_pipeline_1.synthPipeline)((0, scope_stage_1.scopeStages)((0, parse_pipeline_1.parsePipeline)(processingPipeline), "a"))
        : undefined;
    const pipelineScopedToBefore = (0, scope_stage_1.scopeStages)((0, strip_to_gate_1.stripToGate)(pipelineParsed), "b");
    const pipelineScopedToBeforeSynthed = (0, synth_pipeline_1.synthPipeline)(pipelineScopedToBefore);
    const pipelineScopedToBeforeInverted = (0, invert_ppl_1.invertPipeline)(pipelineScopedToBefore).map((ppl) => ppl.map(synth_pipeline_1.synthStage));
    const coll = db.collection((0, derive_pcs_coll_name_1.derivePCSCollName)(collectionName));
    const maxCT = zod_1.default
        .object({
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
    })
        .optional()
        .parse((await coll
        .find({}, { readConcern: mongodb_1.ReadConcernLevel.majority })
        .sort({ ct: -1 })
        .limit(1)
        .project({ _id: 0, ct: 1 })
        .toArray())[0])?.ct;
    if (typeof maxCT === "undefined") {
        // No PCS entries yet
        return;
    }
    const cursorEvent = zod_1.default
        .object({
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
        w: zod_1.default.date().optional(),
    })
        .optional()
        .parse((await coll
        .find({ _id: { $eq: cursor.id } }, { readConcern: mongodb_1.ReadConcernLevel.majority })
        .project({
        _id: 0,
        ct: 1,
        w: 1,
    })
        .toArray())[0]);
    if (typeof cursorEvent === "undefined") {
        throw new CEACursorNotFoundError();
    }
    // The reasoning is that the "tail" of the PCS
    // might be incomplete, as the persister does not
    // transactionally insert all PCS events with the
    // same CT.
    // The same potentially also applies to the "head"
    // of the PCS, as the persister might have started
    // in the middle of a cluster time, but CC already
    // ensures that ccStart > PCS head CT.
    if (!cursorEvent.ct.lt(maxCT)) {
        return;
    }
    if (options?.rejectIfOlderThan) {
        const riot = options.rejectIfOlderThan;
        const cursorEventWLB = 
        // If the event we have fetched has a wall clock, use it
        // Otherwise, find the closest nop event older than the cursor
        cursorEvent.w ??
            zod_1.default
                .array(zod_1.default.object({
                w: zod_1.default.instanceof(Date),
            }))
                .parse(await coll
                .find(
            // The wall clock only exists for nops
            { o: "n", ct: { $lte: cursorEvent.ct } }, { readConcern: mongodb_1.ReadConcernLevel.majority })
                .sort({ ct: -1 })
                .project({ _id: 0, w: 1 })
                .limit(1)
                .toArray())[0]?.w;
        // Prudently, we assume the cursor might be too old
        // if we don't have a lower bound on its wall clock.
        if (!cursorEventWLB || cursorEventWLB < riot) {
            throw new CEACursorTooOldError();
        }
    }
    // As we use the tuple [clusterTime, id] as the sort order,
    // we need two queries.
    // Or one query with an $or clause, but Mongo has trouble
    // planning them optimally.
    const matchRelevantEvents = [
        {
            ct: {
                $eq: cursorEvent.ct,
            },
            _id: { $gt: cursor.id },
        },
        {
            ct: {
                $lt: maxCT,
                $gt: cursorEvent.ct,
            },
        },
    ];
    const idcta = zod_1.default.object({
        _id: zod_1.default.instanceof(mongodb_1.ObjectId),
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
        a: zod_1.default.record(zod_1.default.string(), zod_1.default.unknown()),
    });
    const c1 = (0, stream_algebra_1.streamAppend)(matchRelevantEvents.map((mre) => coll
        .aggregate([
        { $match: { ...mre, o: "i" } },
        ...pipelineScopedToAfter,
        {
            $sort: {
                ct: 1,
                _id: 1,
            },
        },
        ...(processingPipelineScopedToAfter ?? []),
        {
            $project: {
                ct: 1,
                a: 1,
            },
        },
    ], { readConcern: mongodb_1.ReadConcernLevel.majority })
        .map((x) => {
        return { op: "a", ...idcta.parse(x) };
    })[Symbol.asyncIterator]()));
    const c2aschema = zod_1.default
        .object({
        _id: zod_1.default.instanceof(mongodb_1.ObjectId),
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
    })
        .and(zod_1.default
        .object({
        a: zod_1.default.never().optional(),
        u: zod_1.default.record(zod_1.default.string(), zod_1.default.unknown()),
        id: zod_1.default.unknown(),
    })
        .transform((x) => {
        return { op: "u", ...x };
    })
        .or(zod_1.default.object({
        a: zod_1.default.record(zod_1.default.string(), zod_1.default.unknown()),
        u: zod_1.default.never().optional(),
    }))
        .transform((x) => {
        return { op: "r", ...x };
    }));
    const c2a = (0, stream_algebra_1.streamAppend)(matchRelevantEvents.map((mre) => coll
        .aggregate([
        { $match: { ...mre, o: "u" } },
        ...pipelineScopedToAfter,
        ...pipelineScopedToBeforeSynthed,
        {
            $sort: {
                ct: 1,
                _id: 1,
            },
        },
        // We run the processing pipeline even for
        // update events, which throw away the `a`
        // field anyway, as this is easier than
        // creating another stream just for replacement
        // events.
        ...(processingPipelineScopedToAfter ?? []),
        {
            $project: {
                ct: 1,
                u: 1,
                // If we have the update description, we don't
                // need the after image
                a: {
                    $cond: {
                        if: { $ne: ["$u", "$$REMOVE"] },
                        then: "$$REMOVE",
                        else: "$a",
                    },
                },
                // We don't need the object id for replacements
                // It is in the after image anyway
                id: {
                    $cond: {
                        if: { $ne: ["$u", "$$REMOVE"] },
                        then: "$b._id",
                        else: "$$REMOVE",
                    },
                },
            },
        },
    ], { readConcern: mongodb_1.ReadConcernLevel.majority })
        .map((x) => {
        return c2aschema.parse(x);
    })[Symbol.asyncIterator]()));
    const c2bs = pipelineScopedToBeforeInverted.map((ppl) => (0, stream_algebra_1.streamAppend)(matchRelevantEvents.map((mre) => coll
        .aggregate([
        { $match: { ...mre, o: "u" } },
        ...pipelineScopedToAfter,
        ...ppl,
        {
            $sort: {
                ct: 1,
                _id: 1,
            },
        },
        ...(processingPipelineScopedToAfter ?? []),
        {
            $project: {
                ct: 1,
                a: 1,
            },
        },
    ], { readConcern: mongodb_1.ReadConcernLevel.majority })
        .map((x) => {
        return { op: "a", ...idcta.parse(x) };
    })[Symbol.asyncIterator]())));
    const c2cschema = zod_1.default.object({
        _id: zod_1.default.instanceof(mongodb_1.ObjectId),
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
        id: zod_1.default.unknown(),
    });
    const c2c = (0, stream_algebra_1.streamAppend)(matchRelevantEvents.map((mre) => coll
        .aggregate([
        { $match: { ...mre, o: "u" } },
        ...pipelineScopedToBeforeSynthed,
        {
            $sort: {
                ct: 1,
                _id: 1,
            },
        },
        {
            $project: {
                ct: 1,
                id: "$b._id",
            },
        },
    ], { readConcern: mongodb_1.ReadConcernLevel.majority })
        .map((x) => {
        return { op: "s", ...c2cschema.parse(x) };
    })[Symbol.asyncIterator]()));
    const c3schema = zod_1.default.object({
        _id: zod_1.default.instanceof(mongodb_1.ObjectId),
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
        id: zod_1.default.unknown(),
    });
    const c3 = (0, stream_algebra_1.streamAppend)(matchRelevantEvents.map((mre) => coll
        .aggregate([
        { $match: { ...mre, o: "d" } },
        ...pipelineScopedToBeforeSynthed,
        {
            $sort: {
                ct: 1,
                _id: 1,
            },
        },
        {
            $project: {
                ct: 1,
                id: "$b._id",
            },
        },
    ], { readConcern: mongodb_1.ReadConcernLevel.majority })
        .map((x) => {
        return { op: "s", ...c3schema.parse(x) };
    })[Symbol.asyncIterator]()));
    const cs = (0, stream_algebra_1.streamSquashMerge)([
        // additions due to document insertions
        c1,
        // subtractions due to document deletions
        c3,
        // updates
        c2a,
        // additions due to document updates
        // (might need to be cleaned of updates, we do not invert
        // all pipelines)
        ...c2bs,
        // subtractions due to document updates
        // (when cleaned of updates)
        c2c,
    ], pcseLT);
    let lastEventCT;
    for await (const cse of cs) {
        lastEventCT = cse.ct;
        if (cse.op === "u") {
            yield {
                operationType: "update",
                updateDescription: cse.u,
                cursor: {
                    id: cse._id,
                },
                id: cse.id,
                clusterTime: cse.ct,
            };
        }
        else if (cse.op === "r") {
            yield {
                operationType: "replace",
                fullDocument: cse.a,
                cursor: {
                    id: cse._id,
                },
                clusterTime: cse.ct,
            };
        }
        else if (cse.op === "a") {
            yield {
                operationType: "addition",
                fullDocument: cse.a,
                cursor: {
                    id: cse._id,
                },
                clusterTime: cse.ct,
            };
        }
        else {
            cse.op;
            yield {
                operationType: "subtraction",
                cursor: {
                    id: cse._id,
                },
                id: cse.id,
                clusterTime: cse.ct,
            };
        }
    }
    const nopschema = zod_1.default.object({
        _id: zod_1.default.instanceof(mongodb_1.ObjectId),
        ct: zod_1.default.instanceof(mongodb_1.Timestamp),
    });
    yield* (0, stream_algebra_1.streamTake)(1, (0, stream_algebra_1.streamAppend)(matchRelevantEvents.toReversed().map((mre) => coll
        .find({
        $and: [
            mre,
            {
                o: "n",
                ...(lastEventCT
                    ? {
                        ct: {
                            $gt: lastEventCT,
                        },
                    }
                    : {}),
            },
        ],
    }, { readConcern: mongodb_1.ReadConcernLevel.majority })
        .sort({ ct: -1 })
        .limit(1)
        .project({ ct: 1 })
        .map((x) => nopschema.parse(x))
        .map((x) => {
        return {
            operationType: "noop",
            cursor: {
                id: x._id,
            },
            clusterTime: x.ct,
        };
    })[Symbol.asyncIterator]())));
}
class CEACannotResumeError extends Error {
}
exports.CEACannotResumeError = CEACannotResumeError;
class CEACursorNotFoundError extends CEACannotResumeError {
}
exports.CEACursorNotFoundError = CEACursorNotFoundError;
class CEACursorTooOldError extends CEACannotResumeError {
}
exports.CEACursorTooOldError = CEACursorTooOldError;