llamaindex
Version:
<p align="center"> <img height="100" width="100" alt="LlamaIndex logo" src="https://ts.llamaindex.ai/square.svg" /> </p> <h1 align="center">LlamaIndex.TS</h1> <h3 align="center"> Data framework for your LLM application. </h3>
27 lines (26 loc) • 1.03 kB
JavaScript
export async function classify(docStore, nodes) {
const existingDocIds = Object.values(await docStore.getAllDocumentHashes());
const docIdsFromNodes = new Set();
const dedupedNodes = [];
const unusedDocs = [];
for (const node of nodes){
const refDocId = node.sourceNode?.nodeId || node.id_;
docIdsFromNodes.add(refDocId);
const existingHash = await docStore.getDocumentHash(refDocId);
if (!existingHash) {
// document doesn't exist, so add it
dedupedNodes.push(node);
} else if (existingHash && existingHash !== node.hash) {
// document exists but hash is different, so mark doc as unused and add node as deduped
unusedDocs.push(refDocId);
dedupedNodes.push(node);
}
// otherwise, document exists and hash is the same, so do nothing
}
const missingDocs = existingDocIds.filter((id)=>!docIdsFromNodes.has(id));
return {
dedupedNodes,
missingDocs,
unusedDocs
};
}