mongodb-rag-core/build/contentStore/MongoDbPageStore.js

Common elements used by MongoDB Chatbot Framework components.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.makeMongoDbPageStore = void 0;
const pageIdentity_1 = require("./pageIdentity");
const MongoDbDatabaseConnection_1 = require("../MongoDbDatabaseConnection");
/**
  Data store for {@link Page} objects using MongoDB.
 */
function makeMongoDbPageStore({ connectionUri, databaseName, collectionName = "pages", }) {
    const { db, drop, close } = (0, MongoDbDatabaseConnection_1.makeMongoDbDatabaseConnection)({
        connectionUri,
        databaseName,
    });
    const pagesCollection = db.collection(collectionName);
    return {
        queryType: "mongodb",
        drop,
        close,
        metadata: {
            databaseName,
            collectionName,
        },
        async loadPage(args) {
            const filter = args
                ? createQueryFilterFromLoadPagesArgs(args)
                : {};
            return pagesCollection.findOne(filter);
        },
        async loadPages(args) {
            const filter = args
                ? createQueryFilterFromLoadPagesArgs(args)
                : {};
            return pagesCollection.find(filter).toArray();
        },
        async aggregatePages(pipeline) {
            return pagesCollection.aggregate(pipeline).toArray();
        },
        async updatePages(pages) {
            await Promise.all(pages.map(async (page) => {
                const result = await pagesCollection.updateOne((0, pageIdentity_1.pageIdentity)(page), { $set: page }, { upsert: true });
                if (!result.acknowledged) {
                    throw new Error(`update pages not acknowledged!`);
                }
                if (!result.modifiedCount && !result.upsertedCount) {
                    throw new Error(`Page ${JSON.stringify((0, pageIdentity_1.pageIdentity)(page))} not updated!`);
                }
            }));
        },
        async deletePages({ dataSources, permanent = false, inverse = false, }) {
            const filter = {
                ...(dataSources
                    ? { sourceName: { [inverse ? "$nin" : "$in"]: dataSources } }
                    : undefined),
            };
            if (permanent) {
                const result = await pagesCollection.deleteMany(filter);
                if (!result.acknowledged) {
                    throw new Error(`Permanent-delete pages not acknowledged!`);
                }
            }
            else {
                const result = await pagesCollection.updateMany(filter, {
                    $set: { action: "deleted" },
                });
                if (!result.acknowledged) {
                    throw new Error(`Soft-delete pages not acknowledged!`);
                }
            }
        },
        async getMissingPagesByUrl(args) {
            const results = await Promise.all(args.expectedUrls.map(async (url) => {
                const page = await this.loadPage({
                    query: {
                        url: {
                            $regex: new RegExp(args.urlTransformer ? args.urlTransformer(url) : url),
                        },
                    },
                });
                return !page ? url : null;
            }));
            return results.filter((url) => url !== null);
        },
        async getDataSourceVersions(args) {
            const pipeline = [
                {
                    $match: {
                        ...(args && { sourceName: { $in: args.dataSources } }), // Filter by data sources if provided
                        action: { $ne: "deleted" }, // Exclude deleted pages
                        "metadata.version.label": { $exists: true }, // Exclude unversioned pages
                    },
                },
                {
                    $group: {
                        _id: "$sourceName", // Group by sourceName
                        versions: {
                            $addToSet: "$metadata.version", // Collect unique versions
                        },
                    },
                },
                {
                    $project: {
                        _id: 0,
                        sourceName: "$_id",
                        versions: {
                            $sortArray: {
                                input: "$versions",
                                sortBy: {
                                    label: 1, // sort by label in ascending order
                                },
                            },
                        },
                    },
                },
                // create an object with sourceName as key and versions as value
                // to be used in the $arrayToObject operator
                {
                    $group: {
                        _id: null,
                        sources: {
                            $push: {
                                k: "$sourceName",
                                v: "$versions",
                            },
                        },
                    },
                },
                {
                    $project: {
                        _id: 0,
                        sourceVersions: {
                            $arrayToObject: "$sources",
                        },
                    },
                },
            ];
            const result = await pagesCollection
                .aggregate(pipeline)
                .next();
            if (!result) {
                return {};
            }
            return result.sourceVersions;
        },
        async init() {
            await pagesCollection.createIndex({ url: 1 });
            await pagesCollection.createIndex({ sourceName: 1 });
        },
    };
}
exports.makeMongoDbPageStore = makeMongoDbPageStore;
function createQueryFilterFromLoadPagesArgs(args) {
    const { query, sources, updated, urls } = args;
    // We use $and to support custom queries along with the other filters.
    // The $and operator requires at least one element, so we add an empty
    // filter.
    const filter = {
        $and: [{}],
    };
    // Handle custom queries
    if (query !== undefined) {
        if (typeof query === "object" && query !== null) {
            filter["$and"].push(query);
        }
        else {
            throw new Error(`Invalid query - MongoDbPageStore expects a MongoDB query filter. Instead, got: ${query}`);
        }
    }
    // Handle other query filters
    if (updated !== undefined) {
        filter["$and"][0].updated = { $gte: updated };
    }
    if (urls !== undefined) {
        filter["$and"][0].url = { $in: urls };
    }
    if (sources !== undefined) {
        filter["$and"][0].sourceName = { $in: sources };
    }
    return filter;
}
//# sourceMappingURL=MongoDbPageStore.js.map