mongodb-rag-core
Version:
Common elements used by MongoDB Chatbot Framework components.
46 lines • 2.02 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.persistPages = exports.updatePages = void 0;
const getChangedPages_1 = require("./getChangedPages");
const promise_pool_1 = require("@supercharge/promise-pool");
const logger_1 = require("../logger");
/**
Fetches pages from data sources and stores those that have changed in the data
store.
*/
const updatePages = async ({ sources, pageStore, concurrencyOptions, }) => {
await promise_pool_1.PromisePool.withConcurrency(concurrencyOptions?.processDataSources ?? 1)
.for(sources)
.process(async (source, index, pool) => {
logger_1.logger.info(`Fetching pages for ${source.name}`);
const pages = await source.fetchPages();
logger_1.logger.info(`${source.name} returned ${pages.length} pages to process`);
if (pages.length === 0) {
// If a flaky data source returns no pages, we would mark all pages in
// that source as deleted. This is probably not wanted.
logger_1.logger.warn(`Expected at least 1 page from ${source.name}. Discarding result.`);
}
await (0, exports.persistPages)({
pages,
store: pageStore,
sourceName: source.name,
});
});
};
exports.updatePages = updatePages;
/**
Persists pages that have changed.
*/
const persistPages = async ({ store, pages, sourceName, }) => {
const oldPages = await store.loadPages({ sources: [sourceName] });
logger_1.logger.info(`${sourceName} had ${oldPages.length} in the store already`);
const { created, updated, deleted } = await (0, getChangedPages_1.getChangedPages)({
oldPages,
newPages: pages,
sourceName,
});
logger_1.logger.info(`${sourceName}: ${deleted.length} deleted / ${created.length} created / ${updated.length} updated`);
await store.updatePages([...deleted, ...created, ...updated]);
};
exports.persistPages = persistPages;
//# sourceMappingURL=updatePages.js.map