@tmlmobilidade/utils/dist/batching/replicate.js

A collection of utility functions and helpers for the TML Mobilidade Go monorepo, providing common functionality for batching operations, caching, HTTP requests, object manipulation, permissions, and more.

/* * */
import { Timer } from '@tmlmobilidade/timer';
/**
 * Copy documents from a source database to a destination database in multiple steps.
 * The goal of this function is to ensure that the destination database has the same documents
 * as the source database. The replication process is designed to be efficient and to minimize
 * the amount of data transferred between the two databases by only syncing the missing documents.
 *
 * 1. First count the total number of documents in both databases to check if they match. This is a
 * crucial optimization step, as it allows us to skip the replication process if both databases already
 * have the same number of documents, which would indicate that they are already in sync.
 * Though, it's important to note that having the same document count does not guarantee
 * that the documents are identical, but it is a quick check to potentially avoid unnecessary replication.
 *
 * 2. If the counts do not match, get the distinct document IDs from both databases and compare them
 * to find out which ones are missing in the destination database. This step is essential to identify
 * the specific documents that need to be replicated, rather than syncing all documents again.
 * Sync only the missing documents from the source database to the destination database,
 *
 * 3. Delete any extra documents in the destination database that are not present in the source database.
 *
 * 4. Run the onComplete callback function if provided. This allows for any additional actions to be performed after
 * the replication process is complete, such as logging, flushing writers or any other necessary cleanup tasks.
 */
export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteDestinationDbFn, distinctDestinationDbFn, distinctSourceDbFn, missingDocumentsSourceDbAsyncIterator, onCompleteCallbackFn, writeSourceDocumentToDestinationDbFn }) {
    //
    const globalTimer = new Timer();
    //
    // Run the count functions for both databases, if enabled, to get the total number
    // of documents that match a given query. This is done to check if the document count
    // is the same for both databases, which would indicate that all documents are already synced.
    const countStepTimer = new Timer();
    const sourceDbCount = await countSourceDbFn();
    const destinationDbCount = await countDestinationDbFn();
    if (sourceDbCount === destinationDbCount) {
        console.info(`MATCH: Found the same number of documents in both databases: ${sourceDbCount} Source = ${destinationDbCount} Destination (${countStepTimer.get()})`);
        return;
    }
    console.info(`MISMATCH: Document count was different for both databases: ${sourceDbCount} Source != ${destinationDbCount} Destination (${countStepTimer.get()})`);
    //
    // If the document count was different, then check which documents are missing.
    // Instead of syncing all documents again, only the missing IDs are synced.
    // This is done to get the distinct values from each database and comparing
    // them to find the missing ones.
    const distinctStepTimer = new Timer();
    const sourceDbDocIds = await distinctSourceDbFn();
    const sourceDbDocIdsUnique = new Set(sourceDbDocIds);
    const destinationDbDocIds = await distinctDestinationDbFn();
    const destinationDbDocIdsUnique = new Set(destinationDbDocIds);
    const missingDocumentIds = sourceDbDocIds.filter((documentId) => !destinationDbDocIdsUnique.has(documentId));
    const extraDocumentIds = destinationDbDocIds.filter(doc => !sourceDbDocIdsUnique.has(doc));
    console.info(`Source Total: ${sourceDbCount} | Source Unique: ${sourceDbDocIdsUnique.size} | Source ▲: ${sourceDbCount - sourceDbDocIdsUnique.size} | Destination Total: ${destinationDbCount} | Destination Unique: ${destinationDbDocIdsUnique.size} | Destination ▲: ${destinationDbCount - destinationDbDocIdsUnique.size} | Destination Missing: ${missingDocumentIds.length} | Destination Extra: ${extraDocumentIds.length} (${distinctStepTimer.get()})`);
    //
    // If there are missing documents, then they are synced.
    // We query the Source database for the missing documents
    // and write them to the Destination database.
    const missingStepTimer = new Timer();
    if (missingDocumentIds.length > 0) {
        for await (const sourceDbDocument of missingDocumentsSourceDbAsyncIterator(missingDocumentIds)) {
            await writeSourceDocumentToDestinationDbFn(sourceDbDocument);
        }
        console.info(`Synced ${missingDocumentIds.length} missing documents to the Destination database. (${missingStepTimer.get()})`);
    }
    //
    // Extra documents in the destination database should be removed,
    // as they are not present in the source database.
    const deleteStepTimer = new Timer();
    if (extraDocumentIds.length > 0 && deleteDestinationDbFn) {
        await deleteDestinationDbFn(extraDocumentIds);
        console.info(`Deleted ${extraDocumentIds.length} extra documents in the Destination database. (${deleteStepTimer.get()})`);
    }
    //
    // After syncing the missing documents,
    // run the onComplete callback function if provided.
    if (onCompleteCallbackFn)
        await onCompleteCallbackFn();
    console.info(`Replication complete (${globalTimer.get()})`);
    //
}