@langchain/core/dist/indexing/base.d.ts.map

Core LangChain.js abstractions and schemas

{"version":3,"file":"base.d.ts","names":["VectorStore","RecordManagerInterface","HashKeyEncoder","DocumentInterface","BaseDocumentLoader","Metadata","Record","IndexingResult","StringOrDocFunc","HashedDocumentInterface","HashedDocumentArgs","_HashedDocument","CleanupMode","IndexOptions","_batch","T","_deduplicateInOrder","_getSourceIdAssigner","_isBaseDocumentLoader","IndexArgs","index","Promise"],"sources":["../../src/indexing/base.d.ts"],"sourcesContent":["import { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface } from \"./record_manager.js\";\nimport { type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\ntype Metadata = Record<string, unknown>;\ntype IndexingResult = {\n    numAdded: number;\n    numDeleted: number;\n    numUpdated: number;\n    numSkipped: number;\n};\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\nexport interface HashedDocumentInterface extends DocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n}\ninterface HashedDocumentArgs {\n    pageContent: string;\n    metadata: Metadata;\n    uid: string;\n}\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport declare class _HashedDocument implements HashedDocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    private keyEncoder;\n    constructor(fields: HashedDocumentArgs);\n    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;\n    private _hashStringToUUID;\n    private _hashNestedDictToUUID;\n}\nexport type CleanupMode = \"full\" | \"incremental\";\nexport type IndexOptions = {\n    /**\n     * The number of documents to index in one batch.\n     */\n    batchSize?: number;\n    /**\n     * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n     * - **Incremental**: Cleans up all documents that haven't been updated AND\n     *   that are associated with source ids that were seen\n     *   during indexing.\n     *   Clean up is done continuously during indexing helping\n     *   to minimize the probability of users seeing duplicated\n     *   content.\n     * - **Full**: Delete all documents that haven to been returned by the loader.\n     *   Clean up runs after all documents have been indexed.\n     *   This means that users may see duplicated content during indexing.\n     * - **undefined**: Do not delete any documents.\n     */\n    cleanup?: CleanupMode;\n    /**\n     * Optional key that helps identify the original source of the document.\n     * Must either be a string representing the key of the source in the metadata\n     * or a function that takes a document and returns a string representing the source.\n     * **Required when cleanup is incremental**.\n     */\n    sourceIdKey?: StringOrDocFunc;\n    /**\n     * Batch size to use when cleaning up documents.\n     */\n    cleanupBatchSize?: number;\n    /**\n     * Force update documents even if they are present in the\n     * record manager. Useful if you are re-indexing with updated embeddings.\n     */\n    forceUpdate?: boolean;\n};\nexport declare function _batch<T>(size: number, iterable: T[]): T[][];\nexport declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];\nexport declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;\ninterface IndexArgs {\n    docsSource: BaseDocumentLoader | DocumentInterface[];\n    recordManager: RecordManagerInterface;\n    vectorStore: VectorStore;\n    options?: IndexOptions;\n}\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport declare function index(args: IndexArgs): Promise<IndexingResult>;\nexport {};\n"],"mappings":";;;;;;;KAKKK,QAAAA,GAAWC;KACXC,cAAAA;EADAF,QAAAA,EAAAA,MAAQ;EACRE,UAAAA,EAAAA,MAAc;EAMdC,UAAAA,EAAAA,MAAe;EACHC,UAAAA,EAAAA,MAAAA;CAAuB;KADnCD,eAAAA,GAOSH,MAAAA,GAAAA,CAAAA,CAAAA,GAAAA,EAPyBF,iBAOzBE,EAAAA,GAAAA,MAAAA,CAAAA;AAEIF,UARDM,uBAAAA,SAAgCN,iBAQ/BA,CAAAA;EAAiB,GARcA,EAAAA,MAAAA;EAAiB,KAAA,CAAA,EAAA,MAAA;EAUxDO,WAAAA,CAAAA,EAAAA,MAAAA;EAUWC,YAAAA,CAAAA,EAAAA,MAAe;EAAA,WAAA,EAAA,MAAA;EAAA,QAMtBN,EApBAA,QAoBAA;EAAQ,eAEEK,EAAAA,EAAAA,IAAAA;EAAkB,UACFR,EAAAA,EArBtBC,iBAqBsBD;;UAnB9BQ,kBAAAA,CAsBwBP;EAAiB,WAAiBQ,EAAAA,MAAAA;EAAe,QAZnCF,EARlCJ,QAQkCI;EAAuB,GAAA,EAAA,MAAA;AAgBvE;AACA;;;;AAyBiC;AAWTK,cArDHH,eAAAA,YAA2BF,uBAqDlB,CAAA;EAAA,GAAA,EAAA,MAAA;EAAA,KAA4BM,CAAAA,EAAAA,MAAAA;EAAC,WAAKA,CAAAA,EAAAA,MAAAA;EAAC,YAAA,CAAA,EAAA,MAAA;EACzCC,WAAAA,EAAAA,MAAAA;EAAmB,QAAA,EAhD7BX,QAgD6B;EAAA,QAAkBI,UAAAA;EAAuB,WAAKA,CAAAA,MAAAA,EA9CjEC,kBA8CiED;EAAuB,qBAAA,CAAA,YAAA,EA7CxEP,cA6CwE,CAAA,EAAA,IAAA;EACxFe,eAAAA,CAAAA,CAAAA,EAAAA,IAAoB;EAAA,UAAA,CAAA,CAAA,EA5C1Bd,iBA4C0B;EAAA,OAAcK,YAAAA,CAAAA,QAAAA,EA3CxBL,iBA2CwBK,EAAAA,GAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EA3CUG,eA2CVH;EAAe,QAAgBL,iBAAAA;EAAiB,QAAA,qBAAA;AAE1G;AACUgB,KA1CEP,WAAAA,GA0CO,MAAA,GAAA,aAAA;AAAA,KAzCPC,YAAAA,GAyCO;EAAA;;;EAEsB,SACxBb,CAAAA,EAAAA,MAAAA;EAAW;AACF;AAqB1B;;;;;AAAuD;;;;;;YAhDzCY;;;;;;;gBAOIJ;;;;;;;;;;;iBAWMM,kCAAkCC,MAAMA;iBACxCC,mBAAAA,kBAAqCP,4BAA4BA;iBACjEQ,oBAAAA,cAAkCT,+BAA+BL;;cAEpEe,4CAA4Cd;UACvDe,SAAAA;cACMf,qBAAqBD;iBAClBF;eACFD;YACHa;;;;;;;;;;;;;;;;;;;;;iBAqBUO,KAAAA,OAAYD,YAAYE,QAAQd"}