@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 13.9 kB
Source Map (JSON)
{"version":3,"file":"myscale.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: MyScaleLibArgs","vectors: number[][]","documents: Document[]","query: number[]","k: number","filter?: this[\"FilterType\"]","queryResult: {\n data: { text: string; metadata: object; dist: number }[];\n }","result: [Document, number][]","Document","texts: string[]","metadatas: object | object[]","docs: Document[]","dimension?: number","data: string[]","str: string","filter?: MyScaleFilter"],"sources":["../../src/vectorstores/myscale.ts"],"sourcesContent":["import * as uuid from \"uuid\";\nimport { ClickHouseClient, createClient } from \"@clickhouse/client\";\n\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\n\n/**\n * Arguments for the MyScaleStore class, which include the host, port,\n * protocol, username, password, index type, index parameters, column map,\n * database, table, and metric.\n */\nexport interface MyScaleLibArgs {\n host: string;\n port: string | number;\n protocol?: string;\n username: string;\n password: string;\n indexType?: string;\n indexParam?: Record<string, string>;\n columnMap?: ColumnMap;\n database?: string;\n table?: string;\n metric?: metric;\n}\n\n/**\n * Mapping of columns in the MyScale database.\n */\nexport interface ColumnMap {\n id: string;\n text: string;\n vector: string;\n metadata: string;\n}\n\n/**\n * Type of metric used in the MyScale database.\n */\nexport type metric = \"L2\" | \"Cosine\" | \"IP\";\n\n/**\n * Type for filtering search results in the MyScale database.\n */\nexport interface MyScaleFilter {\n whereStr: string;\n}\n\n/**\n * Class for interacting with the MyScale database. It extends the\n * VectorStore class and provides methods for adding vectors and\n * documents, searching for similar vectors, and creating instances from\n * texts or documents.\n */\nexport class MyScaleStore extends VectorStore {\n declare FilterType: MyScaleFilter;\n\n private client: ClickHouseClient;\n\n private indexType: string;\n\n private indexParam: Record<string, string>;\n\n private columnMap: ColumnMap;\n\n private database: string;\n\n private table: string;\n\n private metric: metric;\n\n private isInitialized = false;\n\n _vectorstoreType(): string {\n return \"myscale\";\n }\n\n constructor(embeddings: EmbeddingsInterface, args: MyScaleLibArgs) {\n super(embeddings, args);\n\n this.indexType = args.indexType || \"MSTG\";\n this.indexParam = args.indexParam || {};\n this.columnMap = args.columnMap || {\n id: \"id\",\n text: \"text\",\n vector: \"vector\",\n metadata: \"metadata\",\n };\n this.database = args.database || \"default\";\n this.table = args.table || \"vector_table\";\n this.metric = args.metric || \"Cosine\";\n\n this.client = createClient({\n host: `${args.protocol ?? \"https://\"}${args.host}:${args.port}`,\n username: args.username,\n password: args.password,\n session_id: uuid.v4(),\n });\n }\n\n /**\n * Method to add vectors to the MyScale database.\n * @param vectors The vectors to add.\n * @param documents The documents associated with the vectors.\n * @returns Promise that resolves when the vectors have been added.\n */\n async addVectors(vectors: number[][], documents: Document[]): Promise<void> {\n if (vectors.length === 0) {\n return;\n }\n\n if (!this.isInitialized) {\n await this.initialize(vectors[0].length);\n }\n\n const queryStr = this.buildInsertQuery(vectors, documents);\n await this.client.exec({ query: queryStr });\n }\n\n /**\n * Method to add documents to the MyScale database.\n * @param documents The documents to add.\n * @returns Promise that resolves when the documents have been added.\n */\n async addDocuments(documents: Document[]): Promise<void> {\n return this.addVectors(\n await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),\n documents\n );\n }\n\n /**\n * Method to search for vectors that are similar to a given query vector.\n * @param query The query vector.\n * @param k The number of similar vectors to return.\n * @param filter Optional filter for the search results.\n * @returns Promise that resolves with an array of tuples, each containing a Document and a score.\n */\n async similaritySearchVectorWithScore(\n query: number[],\n k: number,\n filter?: this[\"FilterType\"]\n ): Promise<[Document, number][]> {\n if (!this.isInitialized) {\n await this.initialize(query.length);\n }\n const queryStr = this.buildSearchQuery(query, k, filter);\n\n const queryResultSet = await this.client.query({ query: queryStr });\n const queryResult: {\n data: { text: string; metadata: object; dist: number }[];\n } = await queryResultSet.json();\n\n const result: [Document, number][] = queryResult.data.map((item) => [\n new Document({ pageContent: item.text, metadata: item.metadata }),\n item.dist,\n ]);\n\n return result;\n }\n\n /**\n * Static method to create an instance of MyScaleStore from texts.\n * @param texts The texts to use.\n * @param metadatas The metadata associated with the texts.\n * @param embeddings The embeddings to use.\n * @param args The arguments for the MyScaleStore.\n * @returns Promise that resolves with a new instance of MyScaleStore.\n */\n static async fromTexts(\n texts: string[],\n metadatas: object | object[],\n embeddings: EmbeddingsInterface,\n args: MyScaleLibArgs\n ): Promise<MyScaleStore> {\n const docs: Document[] = [];\n for (let i = 0; i < texts.length; i += 1) {\n const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n const newDoc = new Document({\n pageContent: texts[i],\n metadata,\n });\n docs.push(newDoc);\n }\n return MyScaleStore.fromDocuments(docs, embeddings, args);\n }\n\n /**\n * Static method to create an instance of MyScaleStore from documents.\n * @param docs The documents to use.\n * @param embeddings The embeddings to use.\n * @param args The arguments for the MyScaleStore.\n * @returns Promise that resolves with a new instance of MyScaleStore.\n */\n static async fromDocuments(\n docs: Document[],\n embeddings: EmbeddingsInterface,\n args: MyScaleLibArgs\n ): Promise<MyScaleStore> {\n const instance = new this(embeddings, args);\n await instance.addDocuments(docs);\n return instance;\n }\n\n /**\n * Static method to create an instance of MyScaleStore from an existing\n * index.\n * @param embeddings The embeddings to use.\n * @param args The arguments for the MyScaleStore.\n * @returns Promise that resolves with a new instance of MyScaleStore.\n */\n static async fromExistingIndex(\n embeddings: EmbeddingsInterface,\n args: MyScaleLibArgs\n ): Promise<MyScaleStore> {\n const instance = new this(embeddings, args);\n\n await instance.initialize();\n return instance;\n }\n\n /**\n * Method to initialize the MyScale database.\n * @param dimension Optional dimension of the vectors.\n * @returns Promise that resolves when the database has been initialized.\n */\n private async initialize(dimension?: number): Promise<void> {\n const dim = dimension ?? (await this.embeddings.embedQuery(\"test\")).length;\n\n let indexParamStr = \"\";\n for (const [key, value] of Object.entries(this.indexParam)) {\n indexParamStr += `, '${key}=${value}'`;\n }\n\n const query = `\n CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(\n ${this.columnMap.id} String,\n ${this.columnMap.text} String,\n ${this.columnMap.vector} Array(Float32),\n ${this.columnMap.metadata} JSON,\n CONSTRAINT cons_vec_len CHECK length(${this.columnMap.vector}) = ${dim},\n VECTOR INDEX vidx ${this.columnMap.vector} TYPE ${this.indexType}('metric_type=${this.metric}'${indexParamStr})\n ) ENGINE = MergeTree ORDER BY ${this.columnMap.id}\n `;\n\n await this.client.exec({ query: \"SET allow_experimental_object_type=1\" });\n await this.client.exec({\n query: \"SET output_format_json_named_tuples_as_objects = 1\",\n });\n await this.client.exec({ query });\n this.isInitialized = true;\n }\n\n /**\n * Method to build an SQL query for inserting vectors and documents into\n * the MyScale database.\n * @param vectors The vectors to insert.\n * @param documents The documents to insert.\n * @returns The SQL query string.\n */\n private buildInsertQuery(vectors: number[][], documents: Document[]): string {\n const columnsStr = Object.values(this.columnMap).join(\", \");\n\n const data: string[] = [];\n for (let i = 0; i < vectors.length; i += 1) {\n const vector = vectors[i];\n const document = documents[i];\n const item = [\n `'${uuid.v4()}'`,\n `'${this.escapeString(document.pageContent)}'`,\n `[${vector}]`,\n `'${JSON.stringify(document.metadata)}'`,\n ].join(\", \");\n data.push(`(${item})`);\n }\n const dataStr = data.join(\", \");\n\n return `\n INSERT INTO TABLE\n ${this.database}.${this.table}(${columnsStr})\n VALUES\n ${dataStr}\n `;\n }\n\n private escapeString(str: string): string {\n return str.replace(/\\\\/g, \"\\\\\\\\\").replace(/'/g, \"\\\\'\");\n }\n\n /**\n * Method to build an SQL query for searching for similar vectors in the\n * MyScale database.\n * @param query The query vector.\n * @param k The number of similar vectors to return.\n * @param filter Optional filter for the search results.\n * @returns The SQL query string.\n */\n private buildSearchQuery(\n query: number[],\n k: number,\n filter?: MyScaleFilter\n ): string {\n const order = this.metric === \"IP\" ? \"DESC\" : \"ASC\";\n\n const whereStr = filter ? `PREWHERE ${filter.whereStr}` : \"\";\n return `\n SELECT ${this.columnMap.text} AS text, ${this.columnMap.metadata} AS metadata, dist\n FROM ${this.database}.${this.table}\n ${whereStr}\n ORDER BY distance(${this.columnMap.vector}, [${query}]) AS dist ${order}\n LIMIT ${k}\n `;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAsDA,IAAa,eAAb,MAAa,qBAAqBA,0CAAY;CAG5C,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ,gBAAgB;CAExB,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAsB;EACjE,MAAM,YAAY,KAAK;EAEvB,KAAK,YAAY,KAAK,aAAa;EACnC,KAAK,aAAa,KAAK,cAAc,CAAE;EACvC,KAAK,YAAY,KAAK,aAAa;GACjC,IAAI;GACJ,MAAM;GACN,QAAQ;GACR,UAAU;EACX;EACD,KAAK,WAAW,KAAK,YAAY;EACjC,KAAK,QAAQ,KAAK,SAAS;EAC3B,KAAK,SAAS,KAAK,UAAU;EAE7B,KAAK,+CAAsB;GACzB,MAAM,GAAG,KAAK,YAAY,aAAa,KAAK,KAAK,CAAC,EAAE,KAAK,MAAM;GAC/D,UAAU,KAAK;GACf,UAAU,KAAK;GACf,YAAY,KAAK,IAAI;EACtB,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,SAAqBC,WAAsC;AAC1E,MAAI,QAAQ,WAAW,EACrB;AAGF,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,QAAQ,GAAG,OAAO;EAG1C,MAAM,WAAW,KAAK,iBAAiB,SAAS,UAAU;EAC1D,MAAM,KAAK,OAAO,KAAK,EAAE,OAAO,SAAU,EAAC;CAC5C;;;;;;CAOD,MAAM,aAAaA,WAAsC;AACvD,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,UAAU,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EACzE,UACD;CACF;;;;;;;;CASD,MAAM,gCACJC,OACAC,GACAC,QAC+B;AAC/B,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,MAAM,OAAO;EAErC,MAAM,WAAW,KAAK,iBAAiB,OAAO,GAAG,OAAO;EAExD,MAAM,iBAAiB,MAAM,KAAK,OAAO,MAAM,EAAE,OAAO,SAAU,EAAC;EACnE,MAAMC,cAEF,MAAM,eAAe,MAAM;EAE/B,MAAMC,SAA+B,YAAY,KAAK,IAAI,CAAC,SAAS,CAClE,IAAIC,oCAAS;GAAE,aAAa,KAAK;GAAM,UAAU,KAAK;EAAU,IAChE,KAAK,IACN,EAAC;AAEF,SAAO;CACR;;;;;;;;;CAUD,aAAa,UACXC,OACAC,WACAX,YACAC,MACuB;EACvB,MAAMW,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,SAAS,IAAIH,oCAAS;IAC1B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,OAAO;EAClB;AACD,SAAO,aAAa,cAAc,MAAM,YAAY,KAAK;CAC1D;;;;;;;;CASD,aAAa,cACXG,MACAZ,YACAC,MACuB;EACvB,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;;;;;;;;CASD,aAAa,kBACXD,YACAC,MACuB;EACvB,MAAM,WAAW,IAAI,KAAK,YAAY;EAEtC,MAAM,SAAS,YAAY;AAC3B,SAAO;CACR;;;;;;CAOD,MAAc,WAAWY,WAAmC;EAC1D,MAAM,MAAM,cAAc,MAAM,KAAK,WAAW,WAAW,OAAO,EAAE;EAEpE,IAAI,gBAAgB;AACpB,OAAK,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,QAAQ,KAAK,WAAW,EACxD,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;EAGxC,MAAM,QAAQ,CAAC;iCACc,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;QACvD,EAAE,KAAK,UAAU,GAAG;QACpB,EAAE,KAAK,UAAU,KAAK;QACtB,EAAE,KAAK,UAAU,OAAO;QACxB,EAAE,KAAK,UAAU,SAAS;6CACW,EAAE,KAAK,UAAU,OAAO,IAAI,EAAE,IAAI;0BACrD,EAAE,KAAK,UAAU,OAAO,MAAM,EAAE,KAAK,UAAU,cAAc,EAAE,KAAK,OAAO,CAAC,EAAE,cAAc;oCAClF,EAAE,KAAK,UAAU,GAAG;IACpD,CAAC;EAED,MAAM,KAAK,OAAO,KAAK,EAAE,OAAO,uCAAwC,EAAC;EACzE,MAAM,KAAK,OAAO,KAAK,EACrB,OAAO,qDACR,EAAC;EACF,MAAM,KAAK,OAAO,KAAK,EAAE,MAAO,EAAC;EACjC,KAAK,gBAAgB;CACtB;;;;;;;;CASD,AAAQ,iBAAiBX,SAAqBC,WAA+B;EAC3E,MAAM,aAAa,OAAO,OAAO,KAAK,UAAU,CAAC,KAAK,KAAK;EAE3D,MAAMW,OAAiB,CAAE;AACzB,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GAAG;GAC1C,MAAM,SAAS,QAAQ;GACvB,MAAM,WAAW,UAAU;GAC3B,MAAM,OAAO;IACX,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;IAChB,CAAC,CAAC,EAAE,KAAK,aAAa,SAAS,YAAY,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACb,CAAC,CAAC,EAAE,KAAK,UAAU,SAAS,SAAS,CAAC,CAAC,CAAC;GACzC,EAAC,KAAK,KAAK;GACZ,KAAK,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;EACvB;EACD,MAAM,UAAU,KAAK,KAAK,KAAK;AAE/B,SAAO,CAAC;;QAEJ,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,WAAW;;QAE5C,EAAE,QAAQ;IACd,CAAC;CACF;CAED,AAAQ,aAAaC,KAAqB;AACxC,SAAO,IAAI,QAAQ,OAAO,OAAO,CAAC,QAAQ,MAAM,MAAM;CACvD;;;;;;;;;CAUD,AAAQ,iBACNX,OACAC,GACAW,QACQ;EACR,MAAM,QAAQ,KAAK,WAAW,OAAO,SAAS;EAE9C,MAAM,WAAW,SAAS,CAAC,SAAS,EAAE,OAAO,UAAU,GAAG;AAC1D,SAAO,CAAC;aACC,EAAE,KAAK,UAAU,KAAK,UAAU,EAAE,KAAK,UAAU,SAAS;WAC5D,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;MACnC,EAAE,SAAS;wBACO,EAAE,KAAK,UAAU,OAAO,GAAG,EAAE,MAAM,WAAW,EAAE,MAAM;YAClE,EAAE,EAAE;IACZ,CAAC;CACF;AACF"}