mongodb-rag-core/build/contentStore/updateEmbeddedContent.d.ts

Common elements used by MongoDB Chatbot Framework components.

import { ChunkOptions } from "../chunk";
import { EmbeddedContentStore } from "./EmbeddedContent";
import { Embedder } from "../embed";
import { PageStore, PersistedPage } from ".";
export interface EmbedConcurrencyOptions {
    /**
      Number of pages to chunk and embed concurrently.
     */
    processPages?: number;
    /**
      Maximum number of chunks per page to generate chunks for concurrently.
      This includes the creation of the chunk embeddings and any chunk preprocessing.
     */
    createChunks?: number;
}
/**
 Updates embedded content for pages that have changed since a given date or have different chunking hashes.
 
 @param options - The configuration options for updating embedded content
 @param options.since - The date from which to check for content changes
 @param options.embeddedContentStore - The store containing embedded content
 @param options.pageStore - The store containing pages
 @param options.sourceNames - Optional array of source names to filter pages by
 @param options.embedder - The embedder instance used to generate embeddings
 @param options.chunkOptions - Optional configuration for chunking algorithm
 @param options.concurrencyOptions - Optional configuration for controlling concurrency during embedding
 
 @remarks
 The function performs the following steps:
 1. Finds pages with content changes since the specified date
 2. Identifies pages with different chunking hashes from the current algorithm
 3. Processes both sets of pages by either:
 - Deleting embedded content for deleted pages
 - Updating embedded content for created/updated pages
 
 Processing is done with configurable concurrency to manage system resources.
 
 @throws Will throw an error if there are issues accessing stores or processing pages
 
 @returns Promise that resolves when all updates are complete
 */
export declare const updateEmbeddedContent: ({ since, embeddedContentStore, pageStore, sourceNames, embedder, chunkOptions, concurrencyOptions, }: {
    since: Date;
    embeddedContentStore: EmbeddedContentStore;
    pageStore: PageStore;
    embedder: Embedder;
    chunkOptions?: Partial<ChunkOptions> | undefined;
    sourceNames?: string[] | undefined;
    concurrencyOptions?: EmbedConcurrencyOptions | undefined;
}) => Promise<void>;
export declare const updateEmbeddedContentForPage: ({ page, store, embedder, chunkOptions, concurrencyOptions, chunkAlgoHash, }: {
    page: PersistedPage;
    store: EmbeddedContentStore;
    embedder: Embedder;
    chunkOptions?: Partial<ChunkOptions> | undefined;
    concurrencyOptions?: EmbedConcurrencyOptions | undefined;
    chunkAlgoHash: string;
}) => Promise<void>;
//# sourceMappingURL=updateEmbeddedContent.d.ts.map