UNPKG

@langchain/community

Version:

Third-party integrations for LangChain.js

github.com/langchain-ai/langchainjs/tree/main/libs/langchain-community/

langchain-ai/langchainjs

1 lines • 12.7 kB
1
{"version":3,"file":"typesense.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","config: TypesenseConfig","AsyncCaller","data: T[]","collectionName: string","documents: Document[]","vectors: number[][]","objectWithMetadatas: Record<string, unknown>","typesenseRecords:\n | { document?: Record<string, unknown>; vector_distance: number }[]\n | undefined","documents: [Document, number][]","document: Document","vectorPrompt: number[]","k?: number","filter: this[\"FilterType\"]","hit: VectorSearchResponseHit<object>","documentIds: string[]","docs: Document[]","texts: string[]","metadatas: object[]"],"sources":["../../src/vectorstores/typesense.ts"],"sourcesContent":["import type { Client } from \"typesense\";\nimport type { MultiSearchRequestSchema } from \"typesense/lib/Typesense/MultiSearch.js\";\nimport type {\n SearchResponseHit,\n DocumentSchema,\n} from \"typesense/lib/Typesense/Documents.js\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\nimport {\n AsyncCaller,\n AsyncCallerParams,\n} from \"@langchain/core/utils/async_caller\";\n\n/**\n * Interface for the response hit from a vector search in Typesense.\n */\ninterface VectorSearchResponseHit<T extends DocumentSchema>\n extends SearchResponseHit<T> {\n vector_distance?: number;\n}\n\n/**\n * Typesense vector store configuration.\n */\nexport interface TypesenseConfig extends AsyncCallerParams {\n /**\n * Typesense client.\n */\n typesenseClient: Client;\n /**\n * Typesense schema name in which documents will be stored and searched.\n */\n schemaName: string;\n /**\n * Typesense search parameters.\n * @default { q: '*', per_page: 5, query_by: '' }\n */\n searchParams?: MultiSearchRequestSchema;\n /**\n * Column names.\n */\n columnNames?: {\n /**\n * Vector column name.\n * @default 'vec'\n */\n vector?: string;\n /**\n * Page content column name.\n * @default 'text'\n */\n pageContent?: string;\n /**\n * Metadata column names.\n * @default []\n */\n metadataColumnNames?: string[];\n };\n /**\n * Replace default import function.\n * Default import function will update documents if there is a document with the same id.\n * @param data\n * @param collectionName\n */\n import?<T extends Record<string, unknown> = Record<string, unknown>>(\n data: T[],\n collectionName: string\n ): Promise<void>;\n}\n\n/**\n * Typesense vector store.\n */\nexport class Typesense extends VectorStore {\n declare FilterType: Partial<MultiSearchRequestSchema>;\n\n private client: Client;\n\n private schemaName: string;\n\n private searchParams: MultiSearchRequestSchema;\n\n private vectorColumnName: string;\n\n private pageContentColumnName: string;\n\n private metadataColumnNames: string[];\n\n private caller: AsyncCaller;\n\n private import: (\n data: Record<string, unknown>[],\n collectionName: string\n ) => Promise<void>;\n\n _vectorstoreType(): string {\n return \"typesense\";\n }\n\n constructor(embeddings: EmbeddingsInterface, config: TypesenseConfig) {\n super(embeddings, config);\n\n // Assign config values to class properties.\n this.client = config.typesenseClient;\n this.schemaName = config.schemaName;\n this.searchParams = config.searchParams || {\n q: \"*\",\n per_page: 5,\n query_by: \"\",\n };\n this.vectorColumnName = config.columnNames?.vector || \"vec\";\n this.pageContentColumnName = config.columnNames?.pageContent || \"text\";\n this.metadataColumnNames = config.columnNames?.metadataColumnNames || [];\n\n // Assign import function.\n this.import = config.import || this.importToTypesense.bind(this);\n\n this.caller = new AsyncCaller(config);\n }\n\n /**\n * Default function to import data to typesense\n * @param data\n * @param collectionName\n */\n private async importToTypesense<\n T extends Record<string, unknown> = Record<string, unknown>,\n >(data: T[], collectionName: string) {\n const chunkSize = 2000;\n for (let i = 0; i < data.length; i += chunkSize) {\n const chunk = data.slice(i, i + chunkSize);\n\n await this.caller.call(async () => {\n await this.client\n .collections<T>(collectionName)\n .documents()\n .import(chunk, { action: \"emplace\", dirty_values: \"drop\" });\n });\n }\n }\n\n /**\n * Transform documents to Typesense records.\n * @param documents\n * @returns Typesense records.\n */\n _documentsToTypesenseRecords(\n documents: Document[],\n vectors: number[][]\n ): Record<string, unknown>[] {\n const metadatas = documents.map((doc) => doc.metadata);\n\n const typesenseDocuments = documents.map((doc, index) => {\n const metadata = metadatas[index];\n const objectWithMetadatas: Record<string, unknown> = {};\n\n this.metadataColumnNames.forEach((metadataColumnName) => {\n objectWithMetadatas[metadataColumnName] = metadata[metadataColumnName];\n });\n\n return {\n [this.pageContentColumnName]: doc.pageContent,\n [this.vectorColumnName]: vectors[index],\n ...objectWithMetadatas,\n };\n });\n\n return typesenseDocuments;\n }\n\n /**\n * Transform the Typesense records to documents.\n * @param typesenseRecords\n * @returns documents\n */\n _typesenseRecordsToDocuments(\n typesenseRecords:\n | { document?: Record<string, unknown>; vector_distance: number }[]\n | undefined\n ): [Document, number][] {\n const documents: [Document, number][] =\n typesenseRecords?.map((hit) => {\n const objectWithMetadatas: Record<string, unknown> = {};\n const hitDoc = hit.document || {};\n this.metadataColumnNames.forEach((metadataColumnName) => {\n objectWithMetadatas[metadataColumnName] = hitDoc[metadataColumnName];\n });\n\n const document: Document = {\n pageContent: (hitDoc[this.pageContentColumnName] as string) || \"\",\n metadata: objectWithMetadatas,\n };\n return [document, hit.vector_distance];\n }) || [];\n\n return documents;\n }\n\n /**\n * Add documents to the vector store.\n * Will be updated if in the metadata there is a document with the same id if is using the default import function.\n * Metadata will be added in the columns of the schema based on metadataColumnNames.\n * @param documents Documents to add.\n */\n async addDocuments(documents: Document[]) {\n const typesenseDocuments = this._documentsToTypesenseRecords(\n documents,\n await this.embeddings.embedDocuments(\n documents.map((doc) => doc.pageContent)\n )\n );\n await this.import(typesenseDocuments, this.schemaName);\n }\n\n /**\n * Adds vectors to the vector store.\n * @param vectors Vectors to add.\n * @param documents Documents associated with the vectors.\n */\n async addVectors(vectors: number[][], documents: Document[]) {\n const typesenseDocuments = this._documentsToTypesenseRecords(\n documents,\n vectors\n );\n await this.import(typesenseDocuments, this.schemaName);\n }\n\n /**\n * Search for similar documents with their similarity score.\n * @param vectorPrompt vector to search for\n * @param k amount of results to return\n * @returns similar documents with their similarity score\n */\n async similaritySearchVectorWithScore(\n vectorPrompt: number[],\n k?: number,\n filter: this[\"FilterType\"] = {}\n ) {\n const amount = k || this.searchParams.per_page || 5;\n const vector_query = `${this.vectorColumnName}:([${vectorPrompt}], k:${amount})`;\n const typesenseResponse = await this.client.multiSearch.perform(\n {\n searches: [\n {\n ...this.searchParams,\n ...filter,\n per_page: amount,\n vector_query,\n collection: this.schemaName,\n },\n ],\n },\n {}\n );\n const results = typesenseResponse.results[0].hits;\n\n const hits = results?.map((hit: VectorSearchResponseHit<object>) => ({\n document: hit?.document || {},\n vector_distance: hit?.vector_distance || 2,\n })) as\n | { document: Record<string, unknown>; vector_distance: number }[]\n | undefined;\n\n return this._typesenseRecordsToDocuments(hits);\n }\n\n /**\n * Delete documents from the vector store.\n * @param documentIds ids of the documents to delete\n */\n async deleteDocuments(documentIds: string[]) {\n await this.client\n .collections(this.schemaName)\n .documents()\n .delete({\n filter_by: `id:=${documentIds.join(\",\")}`,\n });\n }\n\n /**\n * Create a vector store from documents.\n * @param docs documents\n * @param embeddings embeddings\n * @param config Typesense configuration\n * @returns Typesense vector store\n * @warning You can omit this method, and only use the constructor and addDocuments.\n */\n static async fromDocuments(\n docs: Document[],\n embeddings: EmbeddingsInterface,\n config: TypesenseConfig\n ): Promise<Typesense> {\n const instance = new Typesense(embeddings, config);\n await instance.addDocuments(docs);\n\n return instance;\n }\n\n /**\n * Create a vector store from texts.\n * @param texts\n * @param metadatas\n * @param embeddings\n * @param config\n * @returns Typesense vector store\n */\n static async fromTexts(\n texts: string[],\n metadatas: object[],\n embeddings: EmbeddingsInterface,\n config: TypesenseConfig\n ) {\n const instance = new Typesense(embeddings, config);\n const documents: Document[] = texts.map((text, i) => ({\n pageContent: text,\n metadata: metadatas[i] || {},\n }));\n await instance.addDocuments(documents);\n\n return instance;\n }\n}\n"],"mappings":";;;;;;;;;;AA0EA,IAAa,YAAb,MAAa,kBAAkBA,0CAAY;CAGzC,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAKR,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,QAAyB;EACpE,MAAM,YAAY,OAAO;EAGzB,KAAK,SAAS,OAAO;EACrB,KAAK,aAAa,OAAO;EACzB,KAAK,eAAe,OAAO,gBAAgB;GACzC,GAAG;GACH,UAAU;GACV,UAAU;EACX;EACD,KAAK,mBAAmB,OAAO,aAAa,UAAU;EACtD,KAAK,wBAAwB,OAAO,aAAa,eAAe;EAChE,KAAK,sBAAsB,OAAO,aAAa,uBAAuB,CAAE;EAGxE,KAAK,SAAS,OAAO,UAAU,KAAK,kBAAkB,KAAK,KAAK;EAEhE,KAAK,SAAS,IAAIC,gDAAY;CAC/B;;;;;;CAOD,MAAc,kBAEZC,MAAWC,gBAAwB;EACnC,MAAM,YAAY;AAClB,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,WAAW;GAC/C,MAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,UAAU;GAE1C,MAAM,KAAK,OAAO,KAAK,YAAY;IACjC,MAAM,KAAK,OACR,YAAe,eAAe,CAC9B,WAAW,CACX,OAAO,OAAO;KAAE,QAAQ;KAAW,cAAc;IAAQ,EAAC;GAC9D,EAAC;EACH;CACF;;;;;;CAOD,6BACEC,WACAC,SAC2B;EAC3B,MAAM,YAAY,UAAU,IAAI,CAAC,QAAQ,IAAI,SAAS;EAEtD,MAAM,qBAAqB,UAAU,IAAI,CAAC,KAAK,UAAU;GACvD,MAAM,WAAW,UAAU;GAC3B,MAAMC,sBAA+C,CAAE;GAEvD,KAAK,oBAAoB,QAAQ,CAAC,uBAAuB;IACvD,oBAAoB,sBAAsB,SAAS;GACpD,EAAC;AAEF,UAAO;KACJ,KAAK,wBAAwB,IAAI;KACjC,KAAK,mBAAmB,QAAQ;IACjC,GAAG;GACJ;EACF,EAAC;AAEF,SAAO;CACR;;;;;;CAOD,6BACEC,kBAGsB;EACtB,MAAMC,YACJ,kBAAkB,IAAI,CAAC,QAAQ;GAC7B,MAAMF,sBAA+C,CAAE;GACvD,MAAM,SAAS,IAAI,YAAY,CAAE;GACjC,KAAK,oBAAoB,QAAQ,CAAC,uBAAuB;IACvD,oBAAoB,sBAAsB,OAAO;GAClD,EAAC;GAEF,MAAMG,WAAqB;IACzB,aAAc,OAAO,KAAK,0BAAqC;IAC/D,UAAU;GACX;AACD,UAAO,CAAC,UAAU,IAAI,eAAgB;EACvC,EAAC,IAAI,CAAE;AAEV,SAAO;CACR;;;;;;;CAQD,MAAM,aAAaL,WAAuB;EACxC,MAAM,qBAAqB,KAAK,6BAC9B,WACA,MAAM,KAAK,WAAW,eACpB,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY,CACxC,CACF;EACD,MAAM,KAAK,OAAO,oBAAoB,KAAK,WAAW;CACvD;;;;;;CAOD,MAAM,WAAWC,SAAqBD,WAAuB;EAC3D,MAAM,qBAAqB,KAAK,6BAC9B,WACA,QACD;EACD,MAAM,KAAK,OAAO,oBAAoB,KAAK,WAAW;CACvD;;;;;;;CAQD,MAAM,gCACJM,cACAC,GACAC,SAA6B,CAAE,GAC/B;EACA,MAAM,SAAS,KAAK,KAAK,aAAa,YAAY;EAClD,MAAM,eAAe,GAAG,KAAK,iBAAiB,GAAG,EAAE,aAAa,KAAK,EAAE,OAAO,CAAC,CAAC;EAChF,MAAM,oBAAoB,MAAM,KAAK,OAAO,YAAY,QACtD,EACE,UAAU,CACR;GACE,GAAG,KAAK;GACR,GAAG;GACH,UAAU;GACV;GACA,YAAY,KAAK;EAClB,CACF,EACF,GACD,CAAE,EACH;EACD,MAAM,UAAU,kBAAkB,QAAQ,GAAG;EAE7C,MAAM,OAAO,SAAS,IAAI,CAACC,SAA0C;GACnE,UAAU,KAAK,YAAY,CAAE;GAC7B,iBAAiB,KAAK,mBAAmB;EAC1C,GAAE;AAIH,SAAO,KAAK,6BAA6B,KAAK;CAC/C;;;;;CAMD,MAAM,gBAAgBC,aAAuB;EAC3C,MAAM,KAAK,OACR,YAAY,KAAK,WAAW,CAC5B,WAAW,CACX,OAAO,EACN,WAAW,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,EAAE,CAC1C,EAAC;CACL;;;;;;;;;CAUD,aAAa,cACXC,MACAhB,YACAC,QACoB;EACpB,MAAM,WAAW,IAAI,UAAU,YAAY;EAC3C,MAAM,SAAS,aAAa,KAAK;AAEjC,SAAO;CACR;;;;;;;;;CAUD,aAAa,UACXgB,OACAC,WACAlB,YACAC,QACA;EACA,MAAM,WAAW,IAAI,UAAU,YAAY;EAC3C,MAAMI,YAAwB,MAAM,IAAI,CAAC,MAAM,OAAO;GACpD,aAAa;GACb,UAAU,UAAU,MAAM,CAAE;EAC7B,GAAE;EACH,MAAM,SAAS,aAAa,UAAU;AAEtC,SAAO;CACR;AACF"}