@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 18.5 kB
Source Map (JSON)
{"version":3,"file":"momento_vector_index.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: MomentoVectorIndexLibArgs","numDimensions: number","CreateVectorIndex","vectors: number[][]","documents: Document<Record<string, any>>[]","ids: string[]","documentProps?: DocumentProps","VectorUpsertItemBatch","documents: Document[]","params: DeleteProps","VectorDeleteItemBatch","query: number[]","k: number","ALL_VECTOR_METADATA","VectorSearch","Document","query: string","options: MaxMarginalRelevanceSearchOptions<this[\"FilterType\"]>","VectorSearchAndFetchVectors","texts: string[]","metadatas: object[] | object","dbConfig: MomentoVectorIndexLibArgs","docs: Document[]","metadata: object"],"sources":["../../src/vectorstores/momento_vector_index.ts"],"sourcesContent":["/* eslint-disable no-instanceof/no-instanceof */\n/* eslint-disable @typescript-eslint/no-explicit-any */\nimport {\n ALL_VECTOR_METADATA,\n IVectorIndexClient,\n VectorIndexItem,\n CreateVectorIndex,\n VectorUpsertItemBatch,\n VectorDeleteItemBatch,\n VectorSearch,\n VectorSearchAndFetchVectors,\n} from \"@gomomento/sdk-core\";\nimport * as uuid from \"uuid\";\nimport { Document } from \"@langchain/core/documents\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport {\n MaxMarginalRelevanceSearchOptions,\n VectorStore,\n} from \"@langchain/core/vectorstores\";\nimport { maximalMarginalRelevance } from \"@langchain/core/utils/math\";\n\nexport interface DocumentProps {\n ids: string[];\n}\n\nexport interface MomentoVectorIndexLibArgs {\n /**\n * The Momento Vector Index client.\n */\n client: IVectorIndexClient;\n /**\n * The name of the index to use to store the data.\n * Defaults to \"default\".\n */\n indexName?: string;\n /**\n * The name of the metadata field to use to store the text of the document.\n * Defaults to \"text\".\n */\n textField?: string;\n /**\n * Whether to create the index if it does not already exist.\n * Defaults to true.\n */\n ensureIndexExists?: boolean;\n}\n\nexport interface DeleteProps {\n /**\n * The ids of the documents to delete.\n */\n ids: string[];\n}\n\n/**\n * A vector store that uses the Momento Vector Index.\n *\n * @remarks\n * To sign up for a free Momento account, visit https://console.gomomento.com.\n */\nexport class MomentoVectorIndex extends VectorStore {\n private client: IVectorIndexClient;\n\n private indexName: string;\n\n private textField: string;\n\n private _ensureIndexExists: boolean;\n\n _vectorstoreType(): string {\n return \"momento\";\n }\n\n /**\n * Creates a new `MomentoVectorIndex` instance.\n * @param embeddings The embeddings instance to use to generate embeddings from documents.\n * @param args The arguments to use to configure the vector store.\n */\n constructor(\n embeddings: EmbeddingsInterface,\n args: MomentoVectorIndexLibArgs\n ) {\n super(embeddings, args);\n\n this.embeddings = embeddings;\n this.client = args.client;\n this.indexName = args.indexName ?? \"default\";\n this.textField = args.textField ?? \"text\";\n this._ensureIndexExists = args.ensureIndexExists ?? true;\n }\n\n /**\n * Returns the Momento Vector Index client.\n * @returns The Momento Vector Index client.\n */\n public getClient(): IVectorIndexClient {\n return this.client;\n }\n\n /**\n * Creates the index if it does not already exist.\n * @param numDimensions The number of dimensions of the vectors to be stored in the index.\n * @returns Promise that resolves to true if the index was created, false if it already existed.\n */\n private async ensureIndexExists(numDimensions: number): Promise<boolean> {\n const response = await this.client.createIndex(\n this.indexName,\n numDimensions\n );\n if (response instanceof CreateVectorIndex.Success) {\n return true;\n } else if (response instanceof CreateVectorIndex.AlreadyExists) {\n return false;\n } else if (response instanceof CreateVectorIndex.Error) {\n throw new Error(response.toString());\n } else {\n throw new Error(`Unknown response type: ${response.toString()}`);\n }\n }\n\n /**\n * Converts the documents to a format that can be stored in the index.\n *\n * This is necessary because the Momento Vector Index requires that the metadata\n * be a map of strings to strings.\n * @param vectors The vectors to convert.\n * @param documents The documents to convert.\n * @param ids The ids to convert.\n * @returns The converted documents.\n */\n private prepareItemBatch(\n vectors: number[][],\n documents: Document<Record<string, any>>[],\n ids: string[]\n ): VectorIndexItem[] {\n return vectors.map((vector, idx) => ({\n id: ids[idx],\n vector,\n metadata: {\n ...documents[idx].metadata,\n [this.textField]: documents[idx].pageContent,\n },\n }));\n }\n\n /**\n * Adds vectors to the index.\n *\n * @remarks If the index does not already exist, it will be created if `ensureIndexExists` is true.\n * @param vectors The vectors to add to the index.\n * @param documents The documents to add to the index.\n * @param documentProps The properties of the documents to add to the index, specifically the ids.\n * @returns Promise that resolves when the vectors have been added to the index. Also returns the ids of the\n * documents that were added.\n */\n public async addVectors(\n vectors: number[][],\n documents: Document<Record<string, any>>[],\n documentProps?: DocumentProps\n ): Promise<void | string[]> {\n if (vectors.length === 0) {\n return;\n }\n\n if (documents.length !== vectors.length) {\n throw new Error(\n `Number of vectors (${vectors.length}) does not equal number of documents (${documents.length})`\n );\n }\n\n if (vectors.some((v) => v.length !== vectors[0].length)) {\n throw new Error(\"All vectors must have the same length\");\n }\n\n if (\n documentProps?.ids !== undefined &&\n documentProps.ids.length !== vectors.length\n ) {\n throw new Error(\n `Number of ids (${\n documentProps?.ids?.length || \"null\"\n }) does not equal number of vectors (${vectors.length})`\n );\n }\n\n if (this._ensureIndexExists) {\n await this.ensureIndexExists(vectors[0].length);\n }\n const documentIds = documentProps?.ids ?? documents.map(() => uuid.v4());\n\n const batchSize = 128;\n const numBatches = Math.ceil(vectors.length / batchSize);\n\n // Add each batch of vectors to the index\n for (let i = 0; i < numBatches; i += 1) {\n const [startIndex, endIndex] = [\n i * batchSize,\n Math.min((i + 1) * batchSize, vectors.length),\n ];\n\n const batchVectors = vectors.slice(startIndex, endIndex);\n const batchDocuments = documents.slice(startIndex, endIndex);\n const batchDocumentIds = documentIds.slice(startIndex, endIndex);\n\n // Insert the items to the index\n const response = await this.client.upsertItemBatch(\n this.indexName,\n this.prepareItemBatch(batchVectors, batchDocuments, batchDocumentIds)\n );\n if (response instanceof VectorUpsertItemBatch.Success) {\n continue;\n } else if (response instanceof VectorUpsertItemBatch.Error) {\n throw new Error(response.toString());\n } else {\n throw new Error(`Unknown response type: ${response.toString()}`);\n }\n }\n }\n\n /**\n * Adds vectors to the index. Generates embeddings from the documents\n * using the `Embeddings` instance passed to the constructor.\n * @param documents Array of `Document` instances to be added to the index.\n * @returns Promise that resolves when the documents have been added to the index.\n */\n async addDocuments(\n documents: Document[],\n documentProps?: DocumentProps\n ): Promise<void> {\n const texts = documents.map(({ pageContent }) => pageContent);\n await this.addVectors(\n await this.embeddings.embedDocuments(texts),\n documents,\n documentProps\n );\n }\n\n /**\n * Deletes vectors from the index by id.\n * @param params The parameters to use to delete the vectors, specifically the ids.\n */\n public async delete(params: DeleteProps): Promise<void> {\n const response = await this.client.deleteItemBatch(\n this.indexName,\n params.ids\n );\n if (response instanceof VectorDeleteItemBatch.Success) {\n // pass\n } else if (response instanceof VectorDeleteItemBatch.Error) {\n throw new Error(response.toString());\n } else {\n throw new Error(`Unknown response type: ${response.toString()}`);\n }\n }\n\n /**\n * Searches the index for the most similar vectors to the query vector.\n * @param query The query vector.\n * @param k The number of results to return.\n * @returns Promise that resolves to the documents of the most similar vectors\n * to the query vector.\n */\n public async similaritySearchVectorWithScore(\n query: number[],\n k: number\n ): Promise<[Document<Record<string, any>>, number][]> {\n const response = await this.client.search(this.indexName, query, {\n topK: k,\n metadataFields: ALL_VECTOR_METADATA,\n });\n if (response instanceof VectorSearch.Success) {\n if (response.hits === undefined) {\n return [];\n }\n\n return response.hits().map((hit) => [\n new Document({\n pageContent: hit.metadata[this.textField]?.toString() ?? \"\",\n metadata: Object.fromEntries(\n Object.entries(hit.metadata).filter(\n ([key]) => key !== this.textField\n )\n ),\n }),\n hit.score,\n ]);\n } else if (response instanceof VectorSearch.Error) {\n throw new Error(response.toString());\n } else {\n throw new Error(`Unknown response type: ${response.toString()}`);\n }\n }\n\n /**\n * Return documents selected using the maximal marginal relevance.\n * Maximal marginal relevance optimizes for similarity to the query AND diversity\n * among selected documents.\n *\n * @param {string} query - Text to look up documents similar to.\n * @param {number} options.k - Number of documents to return.\n * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.\n * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,\n * where 0 corresponds to maximum diversity and 1 to minimum diversity.\n * @param {this[\"FilterType\"]} options.filter - Optional filter\n * @param _callbacks\n *\n * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.\n */\n async maxMarginalRelevanceSearch(\n query: string,\n options: MaxMarginalRelevanceSearchOptions<this[\"FilterType\"]>\n ): Promise<Document[]> {\n const queryEmbedding = await this.embeddings.embedQuery(query);\n const response = await this.client.searchAndFetchVectors(\n this.indexName,\n queryEmbedding,\n { topK: options.fetchK ?? 20, metadataFields: ALL_VECTOR_METADATA }\n );\n\n if (response instanceof VectorSearchAndFetchVectors.Success) {\n const hits = response.hits();\n\n // Gather the embeddings of the search results\n const embeddingList = hits.map((hit) => hit.vector);\n\n // Gather the ids of the most relevant results when applying MMR\n const mmrIndexes = maximalMarginalRelevance(\n queryEmbedding,\n embeddingList,\n options.lambda,\n options.k\n );\n\n const finalResult = mmrIndexes.map((index) => {\n const hit = hits[index];\n const { [this.textField]: pageContent, ...metadata } = hit.metadata;\n return new Document({ metadata, pageContent: pageContent as string });\n });\n return finalResult;\n } else if (response instanceof VectorSearchAndFetchVectors.Error) {\n throw new Error(response.toString());\n } else {\n throw new Error(`Unknown response type: ${response.toString()}`);\n }\n }\n\n /**\n * Stores the documents in the index.\n *\n * Converts the documents to vectors using the `Embeddings` instance passed.\n * @param texts The texts to store in the index.\n * @param metadatas The metadata to store in the index.\n * @param embeddings The embeddings instance to use to generate embeddings from the documents.\n * @param dbConfig The configuration to use to instantiate the vector store.\n * @param documentProps The properties of the documents to add to the index, specifically the ids.\n * @returns Promise that resolves to the vector store.\n */\n public static async fromTexts(\n texts: string[],\n metadatas: object[] | object,\n embeddings: EmbeddingsInterface,\n dbConfig: MomentoVectorIndexLibArgs,\n documentProps?: DocumentProps\n ): Promise<MomentoVectorIndex> {\n if (Array.isArray(metadatas) && texts.length !== metadatas.length) {\n throw new Error(\n `Number of texts (${texts.length}) does not equal number of metadatas (${metadatas.length})`\n );\n }\n\n const docs: Document[] = [];\n for (let i = 0; i < texts.length; i += 1) {\n const metadata: object = Array.isArray(metadatas)\n ? metadatas[i]\n : metadatas;\n const newDoc = new Document({\n pageContent: texts[i],\n metadata,\n });\n docs.push(newDoc);\n }\n return await this.fromDocuments(docs, embeddings, dbConfig, documentProps);\n }\n\n /**\n * Stores the documents in the index.\n * @param docs The documents to store in the index.\n * @param embeddings The embeddings instance to use to generate embeddings from the documents.\n * @param dbConfig The configuration to use to instantiate the vector store.\n * @param documentProps The properties of the documents to add to the index, specifically the ids.\n * @returns Promise that resolves to the vector store.\n */\n public static async fromDocuments(\n docs: Document[],\n embeddings: EmbeddingsInterface,\n dbConfig: MomentoVectorIndexLibArgs,\n documentProps?: DocumentProps\n ): Promise<MomentoVectorIndex> {\n const vectorStore = new MomentoVectorIndex(embeddings, dbConfig);\n await vectorStore.addDocuments(docs, documentProps);\n return vectorStore;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA4DA,IAAa,qBAAb,MAAa,2BAA2BA,0CAAY;CAClD,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,mBAA2B;AACzB,SAAO;CACR;;;;;;CAOD,YACEC,YACAC,MACA;EACA,MAAM,YAAY,KAAK;EAEvB,KAAK,aAAa;EAClB,KAAK,SAAS,KAAK;EACnB,KAAK,YAAY,KAAK,aAAa;EACnC,KAAK,YAAY,KAAK,aAAa;EACnC,KAAK,qBAAqB,KAAK,qBAAqB;CACrD;;;;;CAMD,AAAO,YAAgC;AACrC,SAAO,KAAK;CACb;;;;;;CAOD,MAAc,kBAAkBC,eAAyC;EACvE,MAAM,WAAW,MAAM,KAAK,OAAO,YACjC,KAAK,WACL,cACD;AACD,MAAI,oBAAoBC,uCAAkB,QACxC,QAAO;WACE,oBAAoBA,uCAAkB,cAC/C,QAAO;WACE,oBAAoBA,uCAAkB,MAC/C,OAAM,IAAI,MAAM,SAAS,UAAU;MAEnC,OAAM,IAAI,MAAM,CAAC,uBAAuB,EAAE,SAAS,UAAU,EAAE;CAElE;;;;;;;;;;;CAYD,AAAQ,iBACNC,SACAC,WACAC,KACmB;AACnB,SAAO,QAAQ,IAAI,CAAC,QAAQ,SAAS;GACnC,IAAI,IAAI;GACR;GACA,UAAU;IACR,GAAG,UAAU,KAAK;KACjB,KAAK,YAAY,UAAU,KAAK;GAClC;EACF,GAAE;CACJ;;;;;;;;;;;CAYD,MAAa,WACXF,SACAC,WACAE,eAC0B;AAC1B,MAAI,QAAQ,WAAW,EACrB;AAGF,MAAI,UAAU,WAAW,QAAQ,OAC/B,OAAM,IAAI,MACR,CAAC,mBAAmB,EAAE,QAAQ,OAAO,sCAAsC,EAAE,UAAU,OAAO,CAAC,CAAC;AAIpG,MAAI,QAAQ,KAAK,CAAC,MAAM,EAAE,WAAW,QAAQ,GAAG,OAAO,CACrD,OAAM,IAAI,MAAM;AAGlB,MACE,eAAe,QAAQ,UACvB,cAAc,IAAI,WAAW,QAAQ,OAErC,OAAM,IAAI,MACR,CAAC,eAAe,EACd,eAAe,KAAK,UAAU,OAC/B,oCAAoC,EAAE,QAAQ,OAAO,CAAC,CAAC;AAI5D,MAAI,KAAK,oBACP,MAAM,KAAK,kBAAkB,QAAQ,GAAG,OAAO;EAEjD,MAAM,cAAc,eAAe,OAAO,UAAU,IAAI,MAAM,KAAK,IAAI,CAAC;EAExE,MAAM,YAAY;EAClB,MAAM,aAAa,KAAK,KAAK,QAAQ,SAAS,UAAU;AAGxD,OAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK,GAAG;GACtC,MAAM,CAAC,YAAY,SAAS,GAAG,CAC7B,IAAI,WACJ,KAAK,KAAK,IAAI,KAAK,WAAW,QAAQ,OAAO,AAC9C;GAED,MAAM,eAAe,QAAQ,MAAM,YAAY,SAAS;GACxD,MAAM,iBAAiB,UAAU,MAAM,YAAY,SAAS;GAC5D,MAAM,mBAAmB,YAAY,MAAM,YAAY,SAAS;GAGhE,MAAM,WAAW,MAAM,KAAK,OAAO,gBACjC,KAAK,WACL,KAAK,iBAAiB,cAAc,gBAAgB,iBAAiB,CACtE;AACD,OAAI,oBAAoBC,2CAAsB,QAC5C;YACS,oBAAoBA,2CAAsB,MACnD,OAAM,IAAI,MAAM,SAAS,UAAU;OAEnC,OAAM,IAAI,MAAM,CAAC,uBAAuB,EAAE,SAAS,UAAU,EAAE;EAElE;CACF;;;;;;;CAQD,MAAM,aACJC,WACAF,eACe;EACf,MAAM,QAAQ,UAAU,IAAI,CAAC,EAAE,aAAa,KAAK,YAAY;EAC7D,MAAM,KAAK,WACT,MAAM,KAAK,WAAW,eAAe,MAAM,EAC3C,WACA,cACD;CACF;;;;;CAMD,MAAa,OAAOG,QAAoC;EACtD,MAAM,WAAW,MAAM,KAAK,OAAO,gBACjC,KAAK,WACL,OAAO,IACR;AACD,MAAI,oBAAoBC,2CAAsB,SAAS,CAEtD,WAAU,oBAAoBA,2CAAsB,MACnD,OAAM,IAAI,MAAM,SAAS,UAAU;MAEnC,OAAM,IAAI,MAAM,CAAC,uBAAuB,EAAE,SAAS,UAAU,EAAE;CAElE;;;;;;;;CASD,MAAa,gCACXC,OACAC,GACoD;EACpD,MAAM,WAAW,MAAM,KAAK,OAAO,OAAO,KAAK,WAAW,OAAO;GAC/D,MAAM;GACN,gBAAgBC;EACjB,EAAC;AACF,MAAI,oBAAoBC,kCAAa,SAAS;AAC5C,OAAI,SAAS,SAAS,OACpB,QAAO,CAAE;AAGX,UAAO,SAAS,MAAM,CAAC,IAAI,CAAC,QAAQ,CAClC,IAAIC,oCAAS;IACX,aAAa,IAAI,SAAS,KAAK,YAAY,UAAU,IAAI;IACzD,UAAU,OAAO,YACf,OAAO,QAAQ,IAAI,SAAS,CAAC,OAC3B,CAAC,CAAC,IAAI,KAAK,QAAQ,KAAK,UACzB,CACF;GACF,IACD,IAAI,KACL,EAAC;EACH,WAAU,oBAAoBD,kCAAa,MAC1C,OAAM,IAAI,MAAM,SAAS,UAAU;MAEnC,OAAM,IAAI,MAAM,CAAC,uBAAuB,EAAE,SAAS,UAAU,EAAE;CAElE;;;;;;;;;;;;;;;;CAiBD,MAAM,2BACJE,OACAC,SACqB;EACrB,MAAM,iBAAiB,MAAM,KAAK,WAAW,WAAW,MAAM;EAC9D,MAAM,WAAW,MAAM,KAAK,OAAO,sBACjC,KAAK,WACL,gBACA;GAAE,MAAM,QAAQ,UAAU;GAAI,gBAAgBJ;EAAqB,EACpE;AAED,MAAI,oBAAoBK,iDAA4B,SAAS;GAC3D,MAAM,OAAO,SAAS,MAAM;GAG5B,MAAM,gBAAgB,KAAK,IAAI,CAAC,QAAQ,IAAI,OAAO;GAGnD,MAAM,uEACJ,gBACA,eACA,QAAQ,QACR,QAAQ,EACT;GAED,MAAM,cAAc,WAAW,IAAI,CAAC,UAAU;IAC5C,MAAM,MAAM,KAAK;IACjB,MAAM,EAAE,CAAC,KAAK,YAAY,YAAa,GAAG,UAAU,GAAG,IAAI;AAC3D,WAAO,IAAIH,oCAAS;KAAE;KAAuB;IAAuB;GACrE,EAAC;AACF,UAAO;EACR,WAAU,oBAAoBG,iDAA4B,MACzD,OAAM,IAAI,MAAM,SAAS,UAAU;MAEnC,OAAM,IAAI,MAAM,CAAC,uBAAuB,EAAE,SAAS,UAAU,EAAE;CAElE;;;;;;;;;;;;CAaD,aAAoB,UAClBC,OACAC,WACArB,YACAsB,UACAf,eAC6B;AAC7B,MAAI,MAAM,QAAQ,UAAU,IAAI,MAAM,WAAW,UAAU,OACzD,OAAM,IAAI,MACR,CAAC,iBAAiB,EAAE,MAAM,OAAO,sCAAsC,EAAE,UAAU,OAAO,CAAC,CAAC;EAIhG,MAAMgB,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAMC,WAAmB,MAAM,QAAQ,UAAU,GAC7C,UAAU,KACV;GACJ,MAAM,SAAS,IAAIR,oCAAS;IAC1B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,OAAO;EAClB;AACD,SAAO,MAAM,KAAK,cAAc,MAAM,YAAY,UAAU,cAAc;CAC3E;;;;;;;;;CAUD,aAAoB,cAClBO,MACAvB,YACAsB,UACAf,eAC6B;EAC7B,MAAM,cAAc,IAAI,mBAAmB,YAAY;EACvD,MAAM,YAAY,aAAa,MAAM,cAAc;AACnD,SAAO;CACR;AACF"}