llamaindex/dist/ingestion/strategies/DuplicatesStrategy.js

<p align="center"> <img height="100" width="100" alt="LlamaIndex logo" src="https://ts.llamaindex.ai/square.svg" /> </p> <h1 align="center">LlamaIndex.TS</h1> <h3 align="center"> Data framework for your LLM application. </h3>

import { RollbackableTransformComponent } from "./rollback.js";
/**
 * Handle doc store duplicates by checking all hashes.
 */ export class DuplicatesStrategy extends RollbackableTransformComponent {
    docStore;
    constructor(docStore){
        super(async (nodes)=>{
            const hashes = await this.docStore.getAllDocumentHashes();
            const currentHashes = new Set();
            const nodesToRun = [];
            for (const node of nodes){
                if (!(node.hash in hashes) && !currentHashes.has(node.hash)) {
                    await this.docStore.setDocumentHash(node.id_, node.hash);
                    nodesToRun.push(node);
                    currentHashes.add(node.hash);
                }
            }
            await this.docStore.addDocuments(nodesToRun, true);
            return nodesToRun;
        });
        this.docStore = docStore;
    }
}