llamaindex/ingestion/dist/index.d.cts

<p align="center"> <img height="100" width="100" alt="LlamaIndex logo" src="https://ts.llamaindex.ai/square.svg" /> </p> <h1 align="center">LlamaIndex.TS</h1> <h3 align="center"> Data framework for your LLM application. </h3>

import * as _llamaindex_core_schema from '@llamaindex/core/schema';
import { BaseNode, TransformComponent, Document, BaseReader, Metadata } from '@llamaindex/core/schema';
import { BaseKVStore } from '@llamaindex/core/storage/kv-store';
import { BaseDocumentStore } from '@llamaindex/core/storage/doc-store';
import { BaseVectorStore, VectorStoreByType } from '@llamaindex/core/vector-store';

declare function getTransformationHash(nodes: BaseNode[], transform: TransformComponent): string;
declare class IngestionCache {
    collection: string;
    cache: BaseKVStore;
    nodesKey: string;
    constructor(collection?: string);
    put(hash: string, nodes: BaseNode[]): Promise<void>;
    get(hash: string): Promise<BaseNode[] | undefined>;
}

declare class RollbackableTransformComponent extends TransformComponent {
    rollback(docStore: BaseDocumentStore, nodes: BaseNode[]): Promise<void>;
}

/**
 * Handle doc store duplicates by checking all hashes.
 */
declare class DuplicatesStrategy extends RollbackableTransformComponent {
    private docStore;
    constructor(docStore: BaseDocumentStore);
}

/**
 * Handle docstore upserts by checking hashes and ids.
 * Identify missing docs and delete them from docstore and vector store
 */
declare class UpsertsAndDeleteStrategy extends RollbackableTransformComponent {
    protected docStore: BaseDocumentStore;
    protected vectorStores: BaseVectorStore[] | undefined;
    constructor(docStore: BaseDocumentStore, vectorStores?: BaseVectorStore[]);
}

/**
 * Handles doc store upserts by checking hashes and ids.
 */
declare class UpsertsStrategy extends RollbackableTransformComponent {
    protected docStore: BaseDocumentStore;
    protected vectorStores: BaseVectorStore[] | undefined;
    constructor(docStore: BaseDocumentStore, vectorStores?: BaseVectorStore[]);
}

declare function classify(docStore: BaseDocumentStore, nodes: BaseNode[]): Promise<{
    dedupedNodes: BaseNode<_llamaindex_core_schema.Metadata>[];
    missingDocs: string[];
    unusedDocs: string[];
}>;

/**
 * Document de-deduplication strategies work by comparing the hashes or ids stored in the document store.
 * They require a document store to be set which must be persisted across pipeline runs.
 */
declare enum DocStoreStrategy {
    UPSERTS = "upserts",
    DUPLICATES_ONLY = "duplicates_only",
    UPSERTS_AND_DELETE = "upserts_and_delete",
    NONE = "none"
}
declare function createDocStoreStrategy(docStoreStrategy: DocStoreStrategy, docStore?: BaseDocumentStore, vectorStores?: BaseVectorStore[]): RollbackableTransformComponent;

type TransformRunArgs = {
    inPlace?: boolean;
    cache?: IngestionCache;
    docStoreStrategy?: TransformComponent;
};
declare function runTransformations(nodesToRun: BaseNode[], transformations: TransformComponent[], transformOptions?: any, { inPlace, cache, docStoreStrategy }?: TransformRunArgs): Promise<BaseNode[]>;
declare class IngestionPipeline {
    transformations: TransformComponent[];
    documents?: Document[] | undefined;
    reader?: BaseReader | undefined;
    vectorStore?: BaseVectorStore | undefined;
    vectorStores?: VectorStoreByType | undefined;
    docStore?: BaseDocumentStore;
    docStoreStrategy: DocStoreStrategy;
    cache?: IngestionCache | undefined;
    disableCache: boolean;
    private _docStoreStrategy?;
    constructor(init?: Partial<IngestionPipeline>);
    prepareInput(documents?: Document[], nodes?: BaseNode[]): Promise<BaseNode[]>;
    run(args?: any, transformOptions?: any): Promise<BaseNode[]>;
}
declare function addNodesToVectorStores(nodes: BaseNode<Metadata>[], vectorStores: VectorStoreByType, nodesAdded?: (newIds: string[], nodes: BaseNode<Metadata>[], vectorStore: BaseVectorStore) => Promise<void>): Promise<void>;

export { DocStoreStrategy, DuplicatesStrategy, IngestionCache, IngestionPipeline, RollbackableTransformComponent, UpsertsAndDeleteStrategy, UpsertsStrategy, addNodesToVectorStores, classify, createDocStoreStrategy, getTransformationHash, runTransformations };