mongodb-rag-core/build/contentStore/updateTransformedContent.js

Common elements used by MongoDB Chatbot Framework components.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.updateTranformedContentForPage = exports.updateTransformedContent = void 0;
const crypto_1 = require("crypto");
const promise_pool_1 = require("@supercharge/promise-pool");
const logger_1 = require("../logger");
/**
  (Re-)transform content for pages and persist the transformedContent in the TransformedContentStore.
 */
async function updateTransformedContent({ transformedContentStore, pages, transformPage, concurrencyOptions, }) {
    await promise_pool_1.PromisePool.withConcurrency(concurrencyOptions?.processPages ?? 1)
        .for(pages)
        .process(async (page) => {
        await updateTranformedContentForPage({
            store: transformedContentStore,
            page,
            transformPage,
        });
    });
}
exports.updateTransformedContent = updateTransformedContent;
const chunkAlgoHashes = new Map();
const getHashForFunc = (f) => {
    const data = f.toString();
    const existingHash = chunkAlgoHashes.get(data);
    if (existingHash) {
        return existingHash;
    }
    const hash = (0, crypto_1.createHash)("sha256");
    hash.update(data);
    const digest = hash.digest("hex");
    chunkAlgoHashes.set(data, digest);
    return digest;
};
async function updateTranformedContentForPage({ page, store, transformPage, }) {
    const contentChunks = await transformPage(page);
    if (contentChunks.length === 0) {
        // This could happen if source returned a page with no content
        logger_1.logger.warn(`No content for page ${page.sourceName}:${page.url} - deleting any existing content and continuing`);
        await store.deleteContent({ page });
        return;
    }
    const existingContent = await store.loadContent({
        page,
    });
    const transformAlgoHash = getHashForFunc(transformPage);
    const hasContentChunks = contentChunks.length > 0;
    const hasSameNumberOfChunks = contentChunks.length === existingContent.length;
    const hasSameTransformAlgoHash = existingContent.at(0)?.transformAlgoHash === transformAlgoHash;
    const doNotUpdate = hasContentChunks && hasSameNumberOfChunks && hasSameTransformAlgoHash;
    if (doNotUpdate) {
        logger_1.logger.info(`Transformed content for ${page.sourceName}:${page.url} already updated (${existingContent[0].updated}). Skipping transform.`);
        return;
    }
    logger_1.logger.info(`Adding transformed content for: ${page.url}`);
    const transformedContent = contentChunks.map((chunk) => ({
        ...chunk,
        transformAlgoHash,
    }));
    await store.updateContent({
        page,
        transformedContent,
    });
}
exports.updateTranformedContentForPage = updateTranformedContentForPage;
//# sourceMappingURL=updateTransformedContent.js.map