@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 14.5 kB
Source Map (JSON)
{"version":3,"file":"upstash.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: UpstashVectorLibArgs","FakeEmbeddings","AsyncCaller","documents: DocumentInterface[]","options?: { ids?: string[]; useUpstashEmbeddings?: boolean }","vectors: number[][]","options?: { ids?: string[] }","params: UpstashDeleteParams","query: number[] | string","k: number","filter?: this[\"FilterType\"]","options?: { includeVectors: boolean }","queryResult: QueryResult<UpstashQueryMetadata>[]","searchResult: [DocumentInterface, number][]","Document","texts: string[]","metadatas: UpstashMetadata | UpstashMetadata[]","dbConfig: UpstashVectorLibArgs","docs: DocumentInterface[]"],"sources":["../../src/vectorstores/upstash.ts"],"sourcesContent":["import * as uuid from \"uuid\";\nimport { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Index as UpstashIndex, type QueryResult } from \"@upstash/vector\";\nimport { Document, DocumentInterface } from \"@langchain/core/documents\";\nimport { chunkArray } from \"@langchain/core/utils/chunk_array\";\nimport { FakeEmbeddings } from \"@langchain/core/utils/testing\";\n\nimport {\n AsyncCaller,\n AsyncCallerParams,\n} from \"@langchain/core/utils/async_caller\";\n\n/**\n * This interface defines the arguments for the UpstashVectorStore class.\n */\nexport interface UpstashVectorLibArgs extends AsyncCallerParams {\n index: UpstashIndex;\n filter?: string;\n namespace?: string;\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport type UpstashMetadata = Record<string, any>;\n\nexport type UpstashQueryMetadata = UpstashMetadata & {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n _pageContentLC: any;\n};\n\n/**\n * Type that defines the parameters for the delete method.\n * It can either contain the target id(s) or the deleteAll config to reset all the vectors.\n */\nexport type UpstashDeleteParams =\n | {\n ids: string | string[];\n deleteAll?: never;\n }\n | { deleteAll: boolean; ids?: never };\n\nconst CONCURRENT_UPSERT_LIMIT = 1000;\n\n/**\n * The main class that extends the 'VectorStore' class. It provides\n * methods for interacting with Upstash index, such as adding documents,\n * deleting documents, performing similarity search and more.\n */\nexport class UpstashVectorStore extends VectorStore {\n declare FilterType: string;\n\n index: UpstashIndex;\n\n caller: AsyncCaller;\n\n useUpstashEmbeddings?: boolean;\n\n filter?: this[\"FilterType\"];\n\n namespace?: string;\n\n _vectorstoreType(): string {\n return \"upstash\";\n }\n\n constructor(embeddings: EmbeddingsInterface, args: UpstashVectorLibArgs) {\n super(embeddings, args);\n // Special case where the embeddings instance is a FakeEmbeddings instance. In this case, we need to disable \"instanceof\" rule.\n // eslint-disable-next-line no-instanceof/no-instanceof\n if (embeddings instanceof FakeEmbeddings) {\n this.useUpstashEmbeddings = true;\n }\n\n const { index, namespace, ...asyncCallerArgs } = args;\n\n this.index = index;\n this.caller = new AsyncCaller(asyncCallerArgs);\n this.filter = args.filter;\n this.namespace = namespace;\n }\n\n /**\n * This method adds documents to Upstash database. Documents are first converted to vectors\n * using the provided embeddings instance, and then upserted to the database.\n * @param documents Array of Document objects to be added to the database.\n * @param options Optional object containing array of ids for the documents.\n * @returns Promise that resolves with the ids of the provided documents when the upsert operation is done.\n */\n async addDocuments(\n documents: DocumentInterface[],\n options?: { ids?: string[]; useUpstashEmbeddings?: boolean }\n ) {\n const texts = documents.map(({ pageContent }) => pageContent);\n\n if (this.useUpstashEmbeddings || options?.useUpstashEmbeddings) {\n return this._addData(documents, options);\n }\n\n const embeddings = await this.embeddings.embedDocuments(texts);\n\n return this.addVectors(embeddings, documents, options);\n }\n\n /**\n * This method adds the provided vectors to Upstash database.\n * @param vectors Array of vectors to be added to the Upstash database.\n * @param documents Array of Document objects, each associated with a vector.\n * @param options Optional object containing the array of ids foor the vectors.\n * @returns Promise that resolves with the ids of the provided documents when the upsert operation is done.\n */\n async addVectors(\n vectors: number[][],\n documents: DocumentInterface[],\n options?: { ids?: string[] }\n ) {\n const documentIds =\n options?.ids ?? Array.from({ length: vectors.length }, () => uuid.v4());\n\n const upstashVectors = vectors.map((vector, index) => {\n const metadata = {\n _pageContentLC: documents[index].pageContent,\n ...documents[index].metadata,\n };\n\n const id = documentIds[index];\n\n return {\n id,\n vector,\n metadata,\n };\n });\n\n const namespace = this.index.namespace(this.namespace ?? \"\");\n\n const vectorChunks = chunkArray(upstashVectors, CONCURRENT_UPSERT_LIMIT);\n\n const batchRequests = vectorChunks.map((chunk) =>\n this.caller.call(async () => namespace.upsert(chunk))\n );\n\n await Promise.all(batchRequests);\n\n return documentIds;\n }\n\n /**\n * This method adds the provided documents to Upstash database. The pageContent of the documents will be embedded by Upstash Embeddings.\n * @param documents Array of Document objects to be added to the Upstash database.\n * @param options Optional object containing the array of ids for the documents.\n * @returns Promise that resolves with the ids of the provided documents when the upsert operation is done.\n */\n protected async _addData(\n documents: DocumentInterface[],\n options?: { ids?: string[] }\n ) {\n const documentIds =\n options?.ids ?? Array.from({ length: documents.length }, () => uuid.v4());\n\n const upstashVectorsWithData = documents.map((document, index) => {\n const metadata = {\n _pageContentLC: documents[index].pageContent,\n ...documents[index].metadata,\n };\n\n const id = documentIds[index];\n\n return {\n id,\n data: document.pageContent,\n metadata,\n };\n });\n\n const namespace = this.index.namespace(this.namespace ?? \"\");\n const vectorChunks = chunkArray(\n upstashVectorsWithData,\n CONCURRENT_UPSERT_LIMIT\n );\n\n const batchRequests = vectorChunks.map((chunk) =>\n this.caller.call(async () => namespace.upsert(chunk))\n );\n\n await Promise.all(batchRequests);\n\n return documentIds;\n }\n\n /**\n * This method deletes documents from the Upstash database. You can either\n * provide the target ids, or delete all vectors in the database.\n * @param params Object containing either array of ids of the documents or boolean deleteAll.\n * @returns Promise that resolves when the specified documents have been deleted from the database.\n */\n async delete(params: UpstashDeleteParams): Promise<void> {\n const namespace = this.index.namespace(this.namespace ?? \"\");\n if (params.deleteAll) {\n await namespace.reset();\n } else if (params.ids) {\n await namespace.delete(params.ids);\n }\n }\n\n protected async _runUpstashQuery(\n query: number[] | string,\n k: number,\n filter?: this[\"FilterType\"],\n options?: { includeVectors: boolean }\n ) {\n let queryResult: QueryResult<UpstashQueryMetadata>[] = [];\n\n const namespace = this.index.namespace(this.namespace ?? \"\");\n\n if (typeof query === \"string\") {\n queryResult = await namespace.query<UpstashQueryMetadata>({\n data: query,\n topK: k,\n includeMetadata: true,\n filter,\n ...options,\n });\n } else {\n queryResult = await namespace.query<UpstashQueryMetadata>({\n vector: query,\n topK: k,\n includeMetadata: true,\n filter,\n ...options,\n });\n }\n\n return queryResult;\n }\n\n /**\n * This method performs a similarity search in the Upstash database\n * over the existing vectors.\n * @param query Query vector for the similarity search.\n * @param k The number of similar vectors to return as result.\n * @returns Promise that resolves with an array of tuples, each containing\n * Document object and similarity score. The length of the result will be\n * maximum of 'k' and vectors in the index.\n */\n async similaritySearchVectorWithScore(\n query: number[] | string,\n k: number,\n filter?: this[\"FilterType\"]\n ): Promise<[DocumentInterface, number][]> {\n const results = await this._runUpstashQuery(query, k, filter);\n\n const searchResult: [DocumentInterface, number][] = results.map((res) => {\n const { _pageContentLC, ...metadata } = (res.metadata ??\n {}) as UpstashQueryMetadata;\n return [\n new Document({\n metadata,\n pageContent: _pageContentLC,\n }),\n res.score,\n ];\n });\n\n return searchResult;\n }\n\n /**\n * This method creates a new UpstashVector instance from an array of texts.\n * The texts are initially converted to Document instances and added to Upstash\n * database.\n * @param texts The texts to create the documents from.\n * @param metadatas The metadata values associated with the texts.\n * @param embeddings Embedding interface of choice, to create the text embeddings.\n * @param dbConfig Object containing the Upstash database configs.\n * @returns Promise that resolves with a new UpstashVector instance.\n */\n static async fromTexts(\n texts: string[],\n metadatas: UpstashMetadata | UpstashMetadata[],\n embeddings: EmbeddingsInterface,\n dbConfig: UpstashVectorLibArgs\n ): Promise<UpstashVectorStore> {\n const docs: DocumentInterface[] = [];\n\n for (let i = 0; i < texts.length; i += 1) {\n const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n const newDocument = new Document({\n pageContent: texts[i],\n metadata,\n });\n docs.push(newDocument);\n }\n\n return this.fromDocuments(docs, embeddings, dbConfig);\n }\n\n /**\n * This method creates a new UpstashVector instance from an array of Document instances.\n * @param docs The docs to be added to Upstash database.\n * @param embeddings Embedding interface of choice, to create the embeddings.\n * @param dbConfig Object containing the Upstash database configs.\n * @returns Promise that resolves with a new UpstashVector instance\n */\n static async fromDocuments(\n docs: DocumentInterface[],\n embeddings: EmbeddingsInterface,\n dbConfig: UpstashVectorLibArgs\n ): Promise<UpstashVectorStore> {\n const instance = new this(embeddings, dbConfig);\n await instance.addDocuments(docs);\n return instance;\n }\n\n /**\n * This method creates a new UpstashVector instance from an existing index.\n * @param embeddings Embedding interface of the choice, to create the embeddings.\n * @param dbConfig Object containing the Upstash database configs.\n * @returns\n */\n static async fromExistingIndex(\n embeddings: EmbeddingsInterface,\n dbConfig: UpstashVectorLibArgs\n ): Promise<UpstashVectorStore> {\n const instance = new this(embeddings, dbConfig);\n return instance;\n }\n}\n"],"mappings":";;;;;;;;;;;AAyCA,MAAM,0BAA0B;;;;;;AAOhC,IAAa,qBAAb,cAAwCA,0CAAY;CAGlD;CAEA;CAEA;CAEA;CAEA;CAEA,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAA4B;EACvE,MAAM,YAAY,KAAK;AAGvB,MAAI,sBAAsBC,+CACxB,KAAK,uBAAuB;EAG9B,MAAM,EAAE,OAAO,UAAW,GAAG,iBAAiB,GAAG;EAEjD,KAAK,QAAQ;EACb,KAAK,SAAS,IAAIC,gDAAY;EAC9B,KAAK,SAAS,KAAK;EACnB,KAAK,YAAY;CAClB;;;;;;;;CASD,MAAM,aACJC,WACAC,SACA;EACA,MAAM,QAAQ,UAAU,IAAI,CAAC,EAAE,aAAa,KAAK,YAAY;AAE7D,MAAI,KAAK,wBAAwB,SAAS,qBACxC,QAAO,KAAK,SAAS,WAAW,QAAQ;EAG1C,MAAM,aAAa,MAAM,KAAK,WAAW,eAAe,MAAM;AAE9D,SAAO,KAAK,WAAW,YAAY,WAAW,QAAQ;CACvD;;;;;;;;CASD,MAAM,WACJC,SACAF,WACAG,SACA;EACA,MAAM,cACJ,SAAS,OAAO,MAAM,KAAK,EAAE,QAAQ,QAAQ,OAAQ,GAAE,MAAM,KAAK,IAAI,CAAC;EAEzE,MAAM,iBAAiB,QAAQ,IAAI,CAAC,QAAQ,UAAU;GACpD,MAAM,WAAW;IACf,gBAAgB,UAAU,OAAO;IACjC,GAAG,UAAU,OAAO;GACrB;GAED,MAAM,KAAK,YAAY;AAEvB,UAAO;IACL;IACA;IACA;GACD;EACF,EAAC;EAEF,MAAM,YAAY,KAAK,MAAM,UAAU,KAAK,aAAa,GAAG;EAE5D,MAAM,kEAA0B,gBAAgB,wBAAwB;EAExE,MAAM,gBAAgB,aAAa,IAAI,CAAC,UACtC,KAAK,OAAO,KAAK,YAAY,UAAU,OAAO,MAAM,CAAC,CACtD;EAED,MAAM,QAAQ,IAAI,cAAc;AAEhC,SAAO;CACR;;;;;;;CAQD,MAAgB,SACdH,WACAG,SACA;EACA,MAAM,cACJ,SAAS,OAAO,MAAM,KAAK,EAAE,QAAQ,UAAU,OAAQ,GAAE,MAAM,KAAK,IAAI,CAAC;EAE3E,MAAM,yBAAyB,UAAU,IAAI,CAAC,UAAU,UAAU;GAChE,MAAM,WAAW;IACf,gBAAgB,UAAU,OAAO;IACjC,GAAG,UAAU,OAAO;GACrB;GAED,MAAM,KAAK,YAAY;AAEvB,UAAO;IACL;IACA,MAAM,SAAS;IACf;GACD;EACF,EAAC;EAEF,MAAM,YAAY,KAAK,MAAM,UAAU,KAAK,aAAa,GAAG;EAC5D,MAAM,kEACJ,wBACA,wBACD;EAED,MAAM,gBAAgB,aAAa,IAAI,CAAC,UACtC,KAAK,OAAO,KAAK,YAAY,UAAU,OAAO,MAAM,CAAC,CACtD;EAED,MAAM,QAAQ,IAAI,cAAc;AAEhC,SAAO;CACR;;;;;;;CAQD,MAAM,OAAOC,QAA4C;EACvD,MAAM,YAAY,KAAK,MAAM,UAAU,KAAK,aAAa,GAAG;AAC5D,MAAI,OAAO,WACT,MAAM,UAAU,OAAO;WACd,OAAO,KAChB,MAAM,UAAU,OAAO,OAAO,IAAI;CAErC;CAED,MAAgB,iBACdC,OACAC,GACAC,QACAC,SACA;EACA,IAAIC,cAAmD,CAAE;EAEzD,MAAM,YAAY,KAAK,MAAM,UAAU,KAAK,aAAa,GAAG;AAE5D,MAAI,OAAO,UAAU,UACnB,cAAc,MAAM,UAAU,MAA4B;GACxD,MAAM;GACN,MAAM;GACN,iBAAiB;GACjB;GACA,GAAG;EACJ,EAAC;OAEF,cAAc,MAAM,UAAU,MAA4B;GACxD,QAAQ;GACR,MAAM;GACN,iBAAiB;GACjB;GACA,GAAG;EACJ,EAAC;AAGJ,SAAO;CACR;;;;;;;;;;CAWD,MAAM,gCACJJ,OACAC,GACAC,QACwC;EACxC,MAAM,UAAU,MAAM,KAAK,iBAAiB,OAAO,GAAG,OAAO;EAE7D,MAAMG,eAA8C,QAAQ,IAAI,CAAC,QAAQ;GACvE,MAAM,EAAE,eAAgB,GAAG,UAAU,GAAI,IAAI,YAC3C,CAAE;AACJ,UAAO,CACL,IAAIC,oCAAS;IACX;IACA,aAAa;GACd,IACD,IAAI,KACL;EACF,EAAC;AAEF,SAAO;CACR;;;;;;;;;;;CAYD,aAAa,UACXC,OACAC,WACAjB,YACAkB,UAC6B;EAC7B,MAAMC,OAA4B,CAAE;AAEpC,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,cAAc,IAAIJ,oCAAS;IAC/B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,YAAY;EACvB;AAED,SAAO,KAAK,cAAc,MAAM,YAAY,SAAS;CACtD;;;;;;;;CASD,aAAa,cACXI,MACAnB,YACAkB,UAC6B;EAC7B,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;;;;;;;CAQD,aAAa,kBACXlB,YACAkB,UAC6B;EAC7B,MAAM,WAAW,IAAI,KAAK,YAAY;AACtC,SAAO;CACR;AACF"}