@langchain/core/dist/indexing/base.d.cts.map

Core LangChain.js abstractions and schemas

{"version":3,"file":"base.d.cts","names":["VectorStore","RecordManagerInterface","HashKeyEncoder","DocumentInterface","BaseDocumentLoader","Metadata","Record","IndexingResult","StringOrDocFunc","HashedDocumentInterface","HashedDocumentArgs","_HashedDocument","CleanupMode","IndexOptions","_batch","T","_deduplicateInOrder","_getSourceIdAssigner","_isBaseDocumentLoader","IndexArgs","index","Promise"],"sources":["../../src/indexing/base.d.ts"],"sourcesContent":["import { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface } from \"./record_manager.js\";\nimport { type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\ntype Metadata = Record<string, unknown>;\ntype IndexingResult = {\n    numAdded: number;\n    numDeleted: number;\n    numUpdated: number;\n    numSkipped: number;\n};\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\nexport interface HashedDocumentInterface extends DocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n}\ninterface HashedDocumentArgs {\n    pageContent: string;\n    metadata: Metadata;\n    uid: string;\n}\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport declare class _HashedDocument implements HashedDocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    private keyEncoder;\n    constructor(fields: HashedDocumentArgs);\n    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;\n    private _hashStringToUUID;\n    private _hashNestedDictToUUID;\n}\nexport type CleanupMode = \"full\" | \"incremental\";\nexport type IndexOptions = {\n    /**\n     * The number of documents to index in one batch.\n     */\n    batchSize?: number;\n    /**\n     * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n     * - **Incremental**: Cleans up all documents that haven't been updated AND\n     *   that are associated with source ids that were seen\n     *   during indexing.\n     *   Clean up is done continuously during indexing helping\n     *   to minimize the probability of users seeing duplicated\n     *   content.\n     * - **Full**: Delete all documents that haven to been returned by the loader.\n     *   Clean up runs after all documents have been indexed.\n     *   This means that users may see duplicated content during indexing.\n     * - **undefined**: Do not delete any documents.\n     */\n    cleanup?: CleanupMode;\n    /**\n     * Optional key that helps identify the original source of the document.\n     * Must either be a string representing the key of the source in the metadata\n     * or a function that takes a document and returns a string representing the source.\n     * **Required when cleanup is incremental**.\n     */\n    sourceIdKey?: StringOrDocFunc;\n    /**\n     * Batch size to use when cleaning up documents.\n     */\n    cleanupBatchSize?: number;\n    /**\n     * Force update documents even if they are present in the\n     * record manager. Useful if you are re-indexing with updated embeddings.\n     */\n    forceUpdate?: boolean;\n};\nexport declare function _batch<T>(size: number, iterable: T[]): T[][];\nexport declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];\nexport declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;\nexport declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;\ninterface IndexArgs {\n    docsSource: BaseDocumentLoader | DocumentInterface[];\n    recordManager: RecordManagerInterface;\n    vectorStore: VectorStore;\n    options?: IndexOptions;\n}\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport declare function index(args: IndexArgs): Promise<IndexingResult>;\nexport {};\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;KAKKK,QAAAA,GAAWC;KACXC,cAAAA;EADAF,QAAAA,EAAAA,MAAQ;EACRE,UAAAA,EAAAA,MAAc;EAMdC,UAAAA,EAAAA,MAAe;EACHC,UAAAA,EAAAA,MAAAA;CAMHJ;KAPTG,eAAAA,GASaL,MAAAA,GAAAA,CAAAA,CAAAA,GAAAA,EATqBA,iBASrBA,EAAAA,GAAAA,MAAAA,CAAAA;AAR+BA,UAAhCM,uBAAAA,SAAgCN,iBAAAA,CAAAA;EAAiB,GAAA,EAAA,MAAA;EAUxDO,KAAAA,CAAAA,EAAAA,MAAAA;EAUWC,WAAAA,CAAAA,EAAAA,MAAe;EAMtBN,YAAAA,CAAAA,EAAAA,MAAAA;EAEUK,WAAAA,EAAAA,MAAAA;EACgBR,QAAAA,EAvB1BG,QAuB0BH;EAEtBC,eAAAA,EAAAA,EAAAA,IAAAA;EACgBA,UAAAA,EAAAA,EAxBhBA,iBAwBgBA;;UAtBxBO,kBAAAA,CAUsCD;EAAuB,WAAA,EAAA,MAAA;EAgB3DG,QAAAA,EAxBEP,QAwBS;EACXQ,GAAAA,EAAAA,MAAAA;AAoCZ;AACA;AACA;AACA;AAAoF;;AAE/CV,cA1DhBQ,eAAAA,YAA2BF,uBA0DXN,CAAAA;EAClBF,GAAAA,EAAAA,MAAAA;EACFD,KAAAA,CAAAA,EAAAA,MAAAA;EACHa,WAAAA,CAAAA,EAAAA,MAAAA;EAAY,YAAA,CAAA,EAAA,MAAA;EAqBFO,WAAK,EAAA,MAAA;EAAOD,QAAAA,EA5EtBd,QA4EsBc;EAAoBZ,QAAAA,UAAAA;EAARc,WAAAA,CAAAA,MAAAA,EA1ExBX,kBA0EwBW;EAAO,qBAAA,CAAA,YAAA,EAzEfnB,cAyEe,CAAA,EAAA,IAAA;;gBAvErCC;gCACgBA,kCAAkCQ;;;;KAIxDC,WAAAA;KACAC,YAAAA;;;;;;;;;;;;;;;;;;YAkBED;;;;;;;gBAOIJ;;;;;;;;;;;iBAWMM,kCAAkCC,MAAMA;iBACxCC,mBAAAA,kBAAqCP,4BAA4BA;iBACjEQ,oBAAAA,cAAkCT,+BAA+BL;cACpEe,4CAA4Cd;UACvDe,SAAAA;cACMf,qBAAqBD;iBAClBF;eACFD;YACHa;;;;;;;;;;;;;;;;;;;;;iBAqBUO,KAAAA,OAAYD,YAAYE,QAAQd"}