@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 16.5 kB
Source Map (JSON)
{"version":3,"file":"astradb.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: AstraLibArgs","DataAPIClient","AsyncCaller","fromUser?: CreateCollectionOptions<any>","copy: CreateCollectionOptions<any>","vectors: number[][]","documents: Document[]","options?: string[]","error: any","params: AstraDeleteParams","query: number[]","k: number","filter?: CollectionFilter","results: [Document, number][]","Document","query: string","options: MaxMarginalRelevanceSearchOptions<this[\"FilterType\"]>","embeddingList: number[][]","docs: Document[]","doc: Document","texts: string[]","metadatas: object[] | object","dbConfig: AstraLibArgs"],"sources":["../../src/vectorstores/astradb.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport * as uuid from \"uuid\";\n\nimport {\n Collection,\n DataAPIClient,\n CreateCollectionOptions,\n Db,\n InsertManyError,\n} from \"@datastax/astra-db-ts\";\n\nimport {\n AsyncCaller,\n AsyncCallerParams,\n} from \"@langchain/core/utils/async_caller\";\nimport { Document } from \"@langchain/core/documents\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { maximalMarginalRelevance } from \"@langchain/core/utils/math\";\nimport {\n MaxMarginalRelevanceSearchOptions,\n VectorStore,\n} from \"@langchain/core/vectorstores\";\n\nexport type CollectionFilter = Record<string, unknown>;\n\nexport interface AstraLibArgs extends AsyncCallerParams {\n token: string;\n endpoint: string;\n collection: string;\n keyspace?: string;\n idKey?: string;\n contentKey?: string;\n skipCollectionProvisioning?: boolean;\n collectionOptions?: CreateCollectionOptions<any>;\n batchSize?: number;\n}\n\nexport type AstraDeleteParams = {\n ids: string[];\n};\n\nexport class AstraDBVectorStore extends VectorStore {\n declare FilterType: CollectionFilter;\n\n private astraDBClient: Db;\n\n private collectionName: string;\n\n private collection: Collection | undefined;\n\n private collectionOptions: CreateCollectionOptions<any> | undefined;\n\n private readonly idKey: string;\n\n private readonly contentKey: string; // if undefined the entirety of the content aside from the id and embedding will be stored as content\n\n caller: AsyncCaller;\n\n private readonly skipCollectionProvisioning: boolean;\n\n _vectorstoreType(): string {\n return \"astradb\";\n }\n\n constructor(embeddings: EmbeddingsInterface, args: AstraLibArgs) {\n super(embeddings, args);\n\n const {\n token,\n endpoint,\n collection,\n collectionOptions,\n keyspace,\n idKey,\n contentKey,\n skipCollectionProvisioning,\n ...callerArgs\n } = args;\n const dataAPIClient = new DataAPIClient(token, { caller: [\"langchainjs\"] });\n this.astraDBClient = dataAPIClient.db(endpoint, { keyspace });\n this.skipCollectionProvisioning = skipCollectionProvisioning ?? false;\n if (this.skipCollectionProvisioning && collectionOptions) {\n throw new Error(\n \"If 'skipCollectionProvisioning' has been set to true, 'collectionOptions' must not be defined\"\n );\n }\n this.collectionName = collection;\n this.collectionOptions =\n AstraDBVectorStore.applyCollectionOptionsDefaults(collectionOptions);\n this.idKey = idKey ?? \"_id\";\n this.contentKey = contentKey ?? \"text\";\n this.caller = new AsyncCaller(callerArgs);\n\n if (args.batchSize) {\n console.warn(\n \"[WARNING]: `batchSize` is deprecated, and no longer has any effect.\\n`astra-db-ts` > 1.0.0 handles this internally.\"\n );\n }\n }\n\n private static applyCollectionOptionsDefaults(\n fromUser?: CreateCollectionOptions<any>\n ): CreateCollectionOptions<any> {\n const copy: CreateCollectionOptions<any> = fromUser ? { ...fromUser } : {};\n if (copy.checkExists === undefined) {\n copy.checkExists = false;\n }\n if (copy.indexing === undefined) {\n // same default as langchain python AstraDBVectorStore.\n // this enables to create the collection in python/ts and use it in ts/python with default options.\n copy.indexing = { allow: [\"metadata\"] };\n }\n return copy;\n }\n\n /**\n * Create a new collection in your Astra DB vector database and then connects to it.\n * If the collection already exists, it will connect to it as well.\n *\n * @returns Promise that resolves if connected to the collection.\n */\n async initialize(): Promise<void> {\n if (!this.skipCollectionProvisioning) {\n await this.astraDBClient.createCollection(\n this.collectionName,\n this.collectionOptions\n );\n }\n this.collection = await this.astraDBClient.collection(this.collectionName);\n console.debug(\"Connected to Astra DB collection\");\n }\n\n /**\n * Method to save vectors to AstraDB.\n *\n * @param vectors Vectors to save.\n * @param documents The documents associated with the vectors.\n * @returns Promise that resolves when the vectors have been added.\n */\n async addVectors(\n vectors: number[][],\n documents: Document[],\n options?: string[]\n ) {\n if (!this.collection) {\n throw new Error(\"Must connect to a collection before adding vectors\");\n }\n\n const docs = vectors.map((embedding, idx) => ({\n [this.idKey]: options?.[idx] ?? uuid.v4(),\n [this.contentKey]: documents[idx].pageContent,\n $vector: embedding,\n ...documents[idx].metadata,\n }));\n\n let insertResults;\n\n const isInsertManyError = (error: any): error is InsertManyError =>\n error.name === \"InsertManyError\";\n\n try {\n insertResults = await this.collection.insertMany(docs, {\n ordered: false,\n });\n } catch (error) {\n if (isInsertManyError(error)) {\n insertResults = error.partialResult;\n } else {\n throw error;\n }\n }\n\n const insertedIds = insertResults.insertedIds as string[];\n\n if (insertedIds.length !== docs.length) {\n const missingDocs = docs.filter(\n (doc) => !insertedIds.includes(doc[this.idKey])\n );\n\n for (let i = 0; i < missingDocs.length; i += 1) {\n await this.caller.call(async () => {\n await this.collection?.replaceOne(\n { [this.idKey]: missingDocs[i][this.idKey] },\n missingDocs[i]\n );\n });\n }\n }\n }\n\n /**\n * Method that adds documents to AstraDB.\n *\n * @param documents Array of documents to add to AstraDB.\n * @param options Optional ids for the documents.\n * @returns Promise that resolves the documents have been added.\n */\n async addDocuments(documents: Document[], options?: string[]) {\n if (!this.collection) {\n throw new Error(\"Must connect to a collection before adding vectors\");\n }\n\n return this.addVectors(\n await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),\n documents,\n options\n );\n }\n\n /**\n * Method that deletes documents from AstraDB.\n *\n * @param params AstraDeleteParameters for the delete.\n * @returns Promise that resolves when the documents have been deleted.\n */\n async delete(params: AstraDeleteParams) {\n if (!this.collection) {\n throw new Error(\"Must connect to a collection before deleting\");\n }\n\n await this.collection.deleteMany({ [this.idKey]: { $in: params.ids } });\n console.log(`Deleted ${params.ids.length} documents`);\n }\n\n /**\n * Method that performs a similarity search in AstraDB and returns and similarity scores.\n *\n * @param query Query vector for the similarity search.\n * @param k Number of top results to return.\n * @param filter Optional filter to apply to the search.\n * @returns Promise that resolves with an array of documents and their scores.\n */\n async similaritySearchVectorWithScore(\n query: number[],\n k: number,\n filter?: CollectionFilter\n ): Promise<[Document, number][]> {\n if (!this.collection) {\n throw new Error(\"Must connect to a collection before adding vectors\");\n }\n\n const cursor = await this.collection.find(filter ?? {}, {\n sort: { $vector: query },\n limit: k,\n includeSimilarity: true,\n });\n\n const results: [Document, number][] = [];\n for await (const row of cursor) {\n const {\n $similarity: similarity,\n [this.contentKey]: content,\n ...metadata\n } = row;\n\n const doc = new Document({\n pageContent: content as string,\n metadata,\n });\n\n results.push([doc, similarity as number]);\n }\n return results;\n }\n\n /**\n * Return documents selected using the maximal marginal relevance.\n * Maximal marginal relevance optimizes for similarity to the query AND diversity\n * among selected documents.\n *\n * @param {string} query - Text to look up documents similar to.\n * @param {number} options.k - Number of documents to return.\n * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.\n * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,\n * where 0 corresponds to maximum diversity and 1 to minimum diversity.\n * @param {CollectionFilter} options.filter - Optional filter\n *\n * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.\n */\n async maxMarginalRelevanceSearch(\n query: string,\n options: MaxMarginalRelevanceSearchOptions<this[\"FilterType\"]>\n ): Promise<Document[]> {\n if (!this.collection) {\n throw new Error(\"Must connect to a collection before adding vectors\");\n }\n\n const queryEmbedding = await this.embeddings.embedQuery(query);\n\n const cursor = await this.collection.find(options.filter ?? {}, {\n sort: { $vector: queryEmbedding },\n limit: options.k,\n includeSimilarity: true,\n });\n\n const results = (await cursor.toArray()) ?? [];\n const embeddingList: number[][] = results.map(\n (row) => row.$vector as number[]\n );\n\n const mmrIndexes = maximalMarginalRelevance(\n queryEmbedding,\n embeddingList,\n options.lambda,\n options.k\n );\n\n const topMmrMatches = mmrIndexes.map((idx) => results[idx]);\n\n const docs: Document[] = [];\n topMmrMatches.forEach((match) => {\n const { [this.contentKey]: content, ...metadata } = match;\n\n const doc: Document = {\n pageContent: content as string,\n metadata,\n };\n\n docs.push(doc);\n });\n\n return docs;\n }\n\n /**\n * Static method to create an instance of AstraDBVectorStore from texts.\n *\n * @param texts The texts to use.\n * @param metadatas The metadata associated with the texts.\n * @param embeddings The embeddings to use.\n * @param dbConfig The arguments for the AstraDBVectorStore.\n * @returns Promise that resolves with a new instance of AstraDBVectorStore.\n */\n static async fromTexts(\n texts: string[],\n metadatas: object[] | object,\n embeddings: EmbeddingsInterface,\n dbConfig: AstraLibArgs\n ): Promise<AstraDBVectorStore> {\n const docs: Document[] = [];\n for (let i = 0; i < texts.length; i += 1) {\n const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n const doc = new Document({\n pageContent: texts[i],\n metadata,\n });\n docs.push(doc);\n }\n return AstraDBVectorStore.fromDocuments(docs, embeddings, dbConfig);\n }\n\n /**\n * Static method to create an instance of AstraDBVectorStore from documents.\n *\n * @param docs The Documents to use.\n * @param embeddings The embeddings to use.\n * @param dbConfig The arguments for the AstraDBVectorStore.\n * @returns Promise that resolves with a new instance of AstraDBVectorStore.\n */\n static async fromDocuments(\n docs: Document[],\n embeddings: EmbeddingsInterface,\n dbConfig: AstraLibArgs\n ): Promise<AstraDBVectorStore> {\n const instance = new this(embeddings, dbConfig);\n await instance.initialize();\n\n await instance.addDocuments(docs);\n return instance;\n }\n\n /**\n * Static method to create an instance of AstraDBVectorStore from an existing index.\n *\n * @param embeddings The embeddings to use.\n * @param dbConfig The arguments for the AstraDBVectorStore.\n * @returns Promise that resolves with a new instance of AstraDBVectorStore.\n */\n static async fromExistingIndex(\n embeddings: EmbeddingsInterface,\n dbConfig: AstraLibArgs\n ): Promise<AstraDBVectorStore> {\n const instance = new this(embeddings, dbConfig);\n\n await instance.initialize();\n return instance;\n }\n}\n"],"mappings":";;;;;;;;;;;AAyCA,IAAa,qBAAb,MAAa,2BAA2BA,0CAAY;CAGlD,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAiB;CAEjB,AAAiB;CAEjB;CAEA,AAAiB;CAEjB,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAoB;EAC/D,MAAM,YAAY,KAAK;EAEvB,MAAM,EACJ,OACA,UACA,YACA,mBACA,UACA,OACA,YACA,2BACA,GAAG,YACJ,GAAG;EACJ,MAAM,gBAAgB,IAAIC,qCAAc,OAAO,EAAE,QAAQ,CAAC,aAAc,EAAE;EAC1E,KAAK,gBAAgB,cAAc,GAAG,UAAU,EAAE,SAAU,EAAC;EAC7D,KAAK,6BAA6B,8BAA8B;AAChE,MAAI,KAAK,8BAA8B,kBACrC,OAAM,IAAI,MACR;EAGJ,KAAK,iBAAiB;EACtB,KAAK,oBACH,mBAAmB,+BAA+B,kBAAkB;EACtE,KAAK,QAAQ,SAAS;EACtB,KAAK,aAAa,cAAc;EAChC,KAAK,SAAS,IAAIC,gDAAY;AAE9B,MAAI,KAAK,WACP,QAAQ,KACN,sHACD;CAEJ;CAED,OAAe,+BACbC,UAC8B;EAC9B,MAAMC,OAAqC,WAAW,EAAE,GAAG,SAAU,IAAG,CAAE;AAC1E,MAAI,KAAK,gBAAgB,QACvB,KAAK,cAAc;AAErB,MAAI,KAAK,aAAa,QAGpB,KAAK,WAAW,EAAE,OAAO,CAAC,UAAW,EAAE;AAEzC,SAAO;CACR;;;;;;;CAQD,MAAM,aAA4B;AAChC,MAAI,CAAC,KAAK,4BACR,MAAM,KAAK,cAAc,iBACvB,KAAK,gBACL,KAAK,kBACN;EAEH,KAAK,aAAa,MAAM,KAAK,cAAc,WAAW,KAAK,eAAe;EAC1E,QAAQ,MAAM,mCAAmC;CAClD;;;;;;;;CASD,MAAM,WACJC,SACAC,WACAC,SACA;AACA,MAAI,CAAC,KAAK,WACR,OAAM,IAAI,MAAM;EAGlB,MAAM,OAAO,QAAQ,IAAI,CAAC,WAAW,SAAS;IAC3C,KAAK,QAAQ,UAAU,QAAQ,KAAK,IAAI;IACxC,KAAK,aAAa,UAAU,KAAK;GAClC,SAAS;GACT,GAAG,UAAU,KAAK;EACnB,GAAE;EAEH,IAAI;EAEJ,MAAM,oBAAoB,CAACC,UACzB,MAAM,SAAS;AAEjB,MAAI;GACF,gBAAgB,MAAM,KAAK,WAAW,WAAW,MAAM,EACrD,SAAS,MACV,EAAC;EACH,SAAQ,OAAO;AACd,OAAI,kBAAkB,MAAM,EAC1B,gBAAgB,MAAM;OAEtB,OAAM;EAET;EAED,MAAM,cAAc,cAAc;AAElC,MAAI,YAAY,WAAW,KAAK,QAAQ;GACtC,MAAM,cAAc,KAAK,OACvB,CAAC,QAAQ,CAAC,YAAY,SAAS,IAAI,KAAK,OAAO,CAChD;AAED,QAAK,IAAI,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK,GAC3C,MAAM,KAAK,OAAO,KAAK,YAAY;IACjC,MAAM,KAAK,YAAY,WACrB,GAAG,KAAK,QAAQ,YAAY,GAAG,KAAK,OAAQ,GAC5C,YAAY,GACb;GACF,EAAC;EAEL;CACF;;;;;;;;CASD,MAAM,aAAaF,WAAuBC,SAAoB;AAC5D,MAAI,CAAC,KAAK,WACR,OAAM,IAAI,MAAM;AAGlB,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,UAAU,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EACzE,WACA,QACD;CACF;;;;;;;CAQD,MAAM,OAAOE,QAA2B;AACtC,MAAI,CAAC,KAAK,WACR,OAAM,IAAI,MAAM;EAGlB,MAAM,KAAK,WAAW,WAAW,GAAG,KAAK,QAAQ,EAAE,KAAK,OAAO,IAAK,EAAE,EAAC;EACvE,QAAQ,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,OAAO,UAAU,CAAC,CAAC;CACtD;;;;;;;;;CAUD,MAAM,gCACJC,OACAC,GACAC,QAC+B;AAC/B,MAAI,CAAC,KAAK,WACR,OAAM,IAAI,MAAM;EAGlB,MAAM,SAAS,MAAM,KAAK,WAAW,KAAK,UAAU,CAAE,GAAE;GACtD,MAAM,EAAE,SAAS,MAAO;GACxB,OAAO;GACP,mBAAmB;EACpB,EAAC;EAEF,MAAMC,UAAgC,CAAE;AACxC,aAAW,MAAM,OAAO,QAAQ;GAC9B,MAAM,EACJ,aAAa,YACb,CAAC,KAAK,aAAa,QACnB,GAAG,UACJ,GAAG;GAEJ,MAAM,MAAM,IAAIC,oCAAS;IACvB,aAAa;IACb;GACD;GAED,QAAQ,KAAK,CAAC,KAAK,UAAqB,EAAC;EAC1C;AACD,SAAO;CACR;;;;;;;;;;;;;;;CAgBD,MAAM,2BACJC,OACAC,SACqB;AACrB,MAAI,CAAC,KAAK,WACR,OAAM,IAAI,MAAM;EAGlB,MAAM,iBAAiB,MAAM,KAAK,WAAW,WAAW,MAAM;EAE9D,MAAM,SAAS,MAAM,KAAK,WAAW,KAAK,QAAQ,UAAU,CAAE,GAAE;GAC9D,MAAM,EAAE,SAAS,eAAgB;GACjC,OAAO,QAAQ;GACf,mBAAmB;EACpB,EAAC;EAEF,MAAM,UAAW,MAAM,OAAO,SAAS,IAAK,CAAE;EAC9C,MAAMC,gBAA4B,QAAQ,IACxC,CAAC,QAAQ,IAAI,QACd;EAED,MAAM,uEACJ,gBACA,eACA,QAAQ,QACR,QAAQ,EACT;EAED,MAAM,gBAAgB,WAAW,IAAI,CAAC,QAAQ,QAAQ,KAAK;EAE3D,MAAMC,OAAmB,CAAE;EAC3B,cAAc,QAAQ,CAAC,UAAU;GAC/B,MAAM,EAAE,CAAC,KAAK,aAAa,QAAS,GAAG,UAAU,GAAG;GAEpD,MAAMC,MAAgB;IACpB,aAAa;IACb;GACD;GAED,KAAK,KAAK,IAAI;EACf,EAAC;AAEF,SAAO;CACR;;;;;;;;;;CAWD,aAAa,UACXC,OACAC,WACAtB,YACAuB,UAC6B;EAC7B,MAAMJ,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,MAAM,IAAIJ,oCAAS;IACvB,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,IAAI;EACf;AACD,SAAO,mBAAmB,cAAc,MAAM,YAAY,SAAS;CACpE;;;;;;;;;CAUD,aAAa,cACXI,MACAnB,YACAuB,UAC6B;EAC7B,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,YAAY;EAE3B,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;;;;;;;;CASD,aAAa,kBACXvB,YACAuB,UAC6B;EAC7B,MAAM,WAAW,IAAI,KAAK,YAAY;EAEtC,MAAM,SAAS,YAAY;AAC3B,SAAO;CACR;AACF"}