@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 15.4 kB
Source Map (JSON)
{"version":3,"file":"clickhouse.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: ClickHouseLibArgs","vectors: number[][]","documents: Document[]","query: number[]","k: number","filter?: this[\"FilterType\"]","queryResult: {\n data: { document: string; metadata: object; dist: number }[];\n }","result: [Document, number][]","Document","texts: string[]","metadatas: object | object[]","docs: Document[]","dimension?: number","str: string","filter?: ClickHouseFilter","settingStrings: string[]"],"sources":["../../src/vectorstores/clickhouse.ts"],"sourcesContent":["import * as uuid from \"uuid\";\nimport { ClickHouseClient, createClient } from \"@clickhouse/client\";\nimport { format } from \"mysql2\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\n\n/**\n * Arguments for the ClickHouseStore class, which include the host, port,\n * protocol, username, password, index type, index parameters,\n * index query params, column map, database, table.\n */\nexport interface ClickHouseLibArgs {\n host: string;\n port: string | number;\n protocol?: string;\n username: string;\n password: string;\n indexType?: string;\n indexParam?: string | Record<string, number>;\n indexQueryParams?: Record<string, string>;\n columnMap?: ColumnMap;\n database?: string;\n table?: string;\n}\n\n/**\n * Mapping of columns in the ClickHouse database.\n */\nexport interface ColumnMap {\n id: string;\n uuid: string;\n document: string;\n embedding: string;\n metadata: string;\n}\n\n/**\n * Type for filtering search results in the ClickHouse database.\n */\nexport interface ClickHouseFilter {\n whereStr: string;\n}\n\n/**\n * Class for interacting with the ClickHouse database. It extends the\n * VectorStore class and provides methods for adding vectors and\n * documents, searching for similar vectors, and creating instances from\n * texts or documents.\n */\nexport class ClickHouseStore extends VectorStore {\n declare FilterType: ClickHouseFilter;\n\n private client: ClickHouseClient;\n\n private indexType: string;\n\n private indexParam: string | Record<string, number>;\n\n private indexQueryParams: Record<string, string>;\n\n private columnMap: ColumnMap;\n\n private database: string;\n\n private table: string;\n\n private isInitialized = false;\n\n _vectorstoreType(): string {\n return \"clickhouse\";\n }\n\n constructor(embeddings: EmbeddingsInterface, args: ClickHouseLibArgs) {\n super(embeddings, args);\n\n this.indexType = args.indexType || \"annoy\";\n this.indexParam = args.indexParam || { L2Distance: 100 };\n this.indexQueryParams = args.indexQueryParams || {};\n this.columnMap = args.columnMap || {\n id: \"id\",\n document: \"document\",\n embedding: \"embedding\",\n metadata: \"metadata\",\n uuid: \"uuid\",\n };\n this.database = args.database || \"default\";\n this.table = args.table || \"vector_table\";\n\n this.client = createClient({\n host: `${args.protocol ?? \"https://\"}${args.host}:${args.port}`,\n username: args.username,\n password: args.password,\n session_id: uuid.v4(),\n });\n }\n\n /**\n * Method to add vectors to the ClickHouse database.\n * @param vectors The vectors to add.\n * @param documents The documents associated with the vectors.\n * @returns Promise that resolves when the vectors have been added.\n */\n async addVectors(vectors: number[][], documents: Document[]): Promise<void> {\n if (vectors.length === 0) {\n return;\n }\n\n if (!this.isInitialized) {\n await this.initialize(vectors[0].length);\n }\n\n const queryStr = this.buildInsertQuery(vectors, documents);\n await this.client.exec({ query: queryStr });\n }\n\n /**\n * Method to add documents to the ClickHouse database.\n * @param documents The documents to add.\n * @returns Promise that resolves when the documents have been added.\n */\n async addDocuments(documents: Document[]): Promise<void> {\n return this.addVectors(\n await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),\n documents\n );\n }\n\n /**\n * Method to search for vectors that are similar to a given query vector.\n * @param query The query vector.\n * @param k The number of similar vectors to return.\n * @param filter Optional filter for the search results.\n * @returns Promise that resolves with an array of tuples, each containing a Document and a score.\n */\n async similaritySearchVectorWithScore(\n query: number[],\n k: number,\n filter?: this[\"FilterType\"]\n ): Promise<[Document, number][]> {\n if (!this.isInitialized) {\n await this.initialize(query.length);\n }\n const queryStr = this.buildSearchQuery(query, k, filter);\n\n const queryResultSet = await this.client.query({ query: queryStr });\n\n const queryResult: {\n data: { document: string; metadata: object; dist: number }[];\n } = await queryResultSet.json();\n\n const result: [Document, number][] = queryResult.data.map((item) => [\n new Document({ pageContent: item.document, metadata: item.metadata }),\n item.dist,\n ]);\n\n return result;\n }\n\n /**\n * Static method to create an instance of ClickHouseStore from texts.\n * @param texts The texts to use.\n * @param metadatas The metadata associated with the texts.\n * @param embeddings The embeddings to use.\n * @param args The arguments for the ClickHouseStore.\n * @returns Promise that resolves with a new instance of ClickHouseStore.\n */\n static async fromTexts(\n texts: string[],\n metadatas: object | object[],\n embeddings: EmbeddingsInterface,\n args: ClickHouseLibArgs\n ): Promise<ClickHouseStore> {\n const docs: Document[] = [];\n for (let i = 0; i < texts.length; i += 1) {\n const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n const newDoc = new Document({\n pageContent: texts[i],\n metadata,\n });\n docs.push(newDoc);\n }\n return ClickHouseStore.fromDocuments(docs, embeddings, args);\n }\n\n /**\n * Static method to create an instance of ClickHouseStore from documents.\n * @param docs The documents to use.\n * @param embeddings The embeddings to use.\n * @param args The arguments for the ClickHouseStore.\n * @returns Promise that resolves with a new instance of ClickHouseStore.\n */\n static async fromDocuments(\n docs: Document[],\n embeddings: EmbeddingsInterface,\n args: ClickHouseLibArgs\n ): Promise<ClickHouseStore> {\n const instance = new this(embeddings, args);\n await instance.addDocuments(docs);\n return instance;\n }\n\n /**\n * Static method to create an instance of ClickHouseStore from an existing\n * index.\n * @param embeddings The embeddings to use.\n * @param args The arguments for the ClickHouseStore.\n * @returns Promise that resolves with a new instance of ClickHouseStore.\n */\n static async fromExistingIndex(\n embeddings: EmbeddingsInterface,\n args: ClickHouseLibArgs\n ): Promise<ClickHouseStore> {\n const instance = new this(embeddings, args);\n\n await instance.initialize();\n return instance;\n }\n\n /**\n * Method to initialize the ClickHouse database.\n * @param dimension Optional dimension of the vectors.\n * @returns Promise that resolves when the database has been initialized.\n */\n private async initialize(dimension?: number): Promise<void> {\n const dim = dimension ?? (await this.embeddings.embedQuery(\"test\")).length;\n\n let indexParamStr = \"\";\n if (this.indexParam) {\n if (typeof this.indexParam === \"string\") {\n indexParamStr = this.indexParam;\n } else {\n indexParamStr = Object.entries(this.indexParam)\n .map(([key, value]) => `'${key}', ${value}`)\n .join(\", \");\n }\n }\n\n const query = `\n CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(\n ${this.columnMap.id} Nullable(String),\n ${this.columnMap.document} Nullable(String),\n ${this.columnMap.embedding} Array(Float32),\n ${this.columnMap.metadata} JSON,\n ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),\n CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},\n INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000\n ) ENGINE = MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;\n\n await this.client.exec({\n query,\n clickhouse_settings: {\n allow_experimental_object_type: 1,\n allow_experimental_annoy_index: 1,\n },\n });\n this.isInitialized = true;\n }\n\n /**\n * Method to build an SQL query for inserting vectors and documents into\n * the ClickHouse database.\n * @param vectors The vectors to insert.\n * @param documents The documents to insert.\n * @returns The SQL query string.\n */\n private buildInsertQuery(vectors: number[][], documents: Document[]): string {\n const columnsStr = Object.values(\n Object.fromEntries(\n Object.entries(this.columnMap).filter(\n ([key]) => key !== this.columnMap.uuid\n )\n )\n ).join(\", \");\n\n const placeholders = vectors.map(() => \"(?, ?, ?, ?)\").join(\", \");\n const values = [];\n\n for (let i = 0; i < vectors.length; i += 1) {\n const vector = vectors[i];\n const document = documents[i];\n values.push(\n uuid.v4(),\n this.escapeString(document.pageContent),\n JSON.stringify(vector),\n JSON.stringify(document.metadata)\n );\n }\n\n const insertQueryStr = `\n INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr}) \n VALUES ${placeholders}\n `;\n\n const insertQuery = format(insertQueryStr, values);\n return insertQuery;\n }\n\n private escapeString(str: string): string {\n return str.replace(/\\\\/g, \"\\\\\\\\\").replace(/'/g, \"\\\\'\");\n }\n\n /**\n * Method to build an SQL query for searching for similar vectors in the\n * ClickHouse database.\n * @param query The query vector.\n * @param k The number of similar vectors to return.\n * @param filter Optional filter for the search results.\n * @returns The SQL query string.\n */\n private buildSearchQuery(\n query: number[],\n k: number,\n filter?: ClickHouseFilter\n ): string {\n const order = \"ASC\";\n const whereStr = filter ? `PREWHERE ${filter.whereStr}` : \"\";\n const placeholders = query.map(() => \"?\").join(\", \");\n\n const settingStrings: string[] = [];\n if (this.indexQueryParams) {\n for (const [key, value] of Object.entries(this.indexQueryParams)) {\n settingStrings.push(`SETTING ${key}=${value}`);\n }\n }\n\n const searchQueryStr = `\n SELECT ${this.columnMap.document} AS document, ${\n this.columnMap.metadata\n } AS metadata, dist\n FROM ${this.database}.${this.table}\n ${whereStr}\n ORDER BY L2Distance(${\n this.columnMap.embedding\n }, [${placeholders}]) AS dist ${order}\n LIMIT ${k} ${settingStrings.join(\" \")}\n `;\n\n // Format the query with actual values\n const searchQuery = format(searchQueryStr, query);\n return searchQuery;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkDA,IAAa,kBAAb,MAAa,wBAAwBA,0CAAY;CAG/C,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ,gBAAgB;CAExB,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAyB;EACpE,MAAM,YAAY,KAAK;EAEvB,KAAK,YAAY,KAAK,aAAa;EACnC,KAAK,aAAa,KAAK,cAAc,EAAE,YAAY,IAAK;EACxD,KAAK,mBAAmB,KAAK,oBAAoB,CAAE;EACnD,KAAK,YAAY,KAAK,aAAa;GACjC,IAAI;GACJ,UAAU;GACV,WAAW;GACX,UAAU;GACV,MAAM;EACP;EACD,KAAK,WAAW,KAAK,YAAY;EACjC,KAAK,QAAQ,KAAK,SAAS;EAE3B,KAAK,+CAAsB;GACzB,MAAM,GAAG,KAAK,YAAY,aAAa,KAAK,KAAK,CAAC,EAAE,KAAK,MAAM;GAC/D,UAAU,KAAK;GACf,UAAU,KAAK;GACf,YAAY,KAAK,IAAI;EACtB,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,SAAqBC,WAAsC;AAC1E,MAAI,QAAQ,WAAW,EACrB;AAGF,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,QAAQ,GAAG,OAAO;EAG1C,MAAM,WAAW,KAAK,iBAAiB,SAAS,UAAU;EAC1D,MAAM,KAAK,OAAO,KAAK,EAAE,OAAO,SAAU,EAAC;CAC5C;;;;;;CAOD,MAAM,aAAaA,WAAsC;AACvD,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,UAAU,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EACzE,UACD;CACF;;;;;;;;CASD,MAAM,gCACJC,OACAC,GACAC,QAC+B;AAC/B,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,MAAM,OAAO;EAErC,MAAM,WAAW,KAAK,iBAAiB,OAAO,GAAG,OAAO;EAExD,MAAM,iBAAiB,MAAM,KAAK,OAAO,MAAM,EAAE,OAAO,SAAU,EAAC;EAEnE,MAAMC,cAEF,MAAM,eAAe,MAAM;EAE/B,MAAMC,SAA+B,YAAY,KAAK,IAAI,CAAC,SAAS,CAClE,IAAIC,oCAAS;GAAE,aAAa,KAAK;GAAU,UAAU,KAAK;EAAU,IACpE,KAAK,IACN,EAAC;AAEF,SAAO;CACR;;;;;;;;;CAUD,aAAa,UACXC,OACAC,WACAX,YACAC,MAC0B;EAC1B,MAAMW,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,SAAS,IAAIH,oCAAS;IAC1B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,OAAO;EAClB;AACD,SAAO,gBAAgB,cAAc,MAAM,YAAY,KAAK;CAC7D;;;;;;;;CASD,aAAa,cACXG,MACAZ,YACAC,MAC0B;EAC1B,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;;;;;;;;CASD,aAAa,kBACXD,YACAC,MAC0B;EAC1B,MAAM,WAAW,IAAI,KAAK,YAAY;EAEtC,MAAM,SAAS,YAAY;AAC3B,SAAO;CACR;;;;;;CAOD,MAAc,WAAWY,WAAmC;EAC1D,MAAM,MAAM,cAAc,MAAM,KAAK,WAAW,WAAW,OAAO,EAAE;EAEpE,IAAI,gBAAgB;AACpB,MAAI,KAAK,WACP,KAAI,OAAO,KAAK,eAAe,UAC7B,gBAAgB,KAAK;OAErB,gBAAgB,OAAO,QAAQ,KAAK,WAAW,CAC5C,IAAI,CAAC,CAAC,KAAK,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,EAAE,OAAO,CAAC,CAC3C,KAAK,KAAK;EAIjB,MAAM,QAAQ,CAAC;+BACY,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;MACvD,EAAE,KAAK,UAAU,GAAG;MACpB,EAAE,KAAK,UAAU,SAAS;MAC1B,EAAE,KAAK,UAAU,UAAU;MAC3B,EAAE,KAAK,UAAU,SAAS;MAC1B,EAAE,KAAK,UAAU,KAAK;2CACe,EAAE,KAAK,UAAU,UAAU,IAAI,EAAE,IAAI;oBAC5D,EAAE,KAAK,UAAU,UAAU,MAAM,EAAE,KAAK,UAAU,CAAC,EAAE,cAAc;mCACpD,EAAE,KAAK,UAAU,KAAK,mCAAmC,CAAC;EAEzF,MAAM,KAAK,OAAO,KAAK;GACrB;GACA,qBAAqB;IACnB,gCAAgC;IAChC,gCAAgC;GACjC;EACF,EAAC;EACF,KAAK,gBAAgB;CACtB;;;;;;;;CASD,AAAQ,iBAAiBX,SAAqBC,WAA+B;EAC3E,MAAM,aAAa,OAAO,OACxB,OAAO,YACL,OAAO,QAAQ,KAAK,UAAU,CAAC,OAC7B,CAAC,CAAC,IAAI,KAAK,QAAQ,KAAK,UAAU,KACnC,CACF,CACF,CAAC,KAAK,KAAK;EAEZ,MAAM,eAAe,QAAQ,IAAI,MAAM,eAAe,CAAC,KAAK,KAAK;EACjE,MAAM,SAAS,CAAE;AAEjB,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GAAG;GAC1C,MAAM,SAAS,QAAQ;GACvB,MAAM,WAAW,UAAU;GAC3B,OAAO,KACL,KAAK,IAAI,EACT,KAAK,aAAa,SAAS,YAAY,EACvC,KAAK,UAAU,OAAO,EACtB,KAAK,UAAU,SAAS,SAAS,CAClC;EACF;EAED,MAAM,iBAAiB,CAAC;wBACJ,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,WAAW;aACvD,EAAE,aAAa;IACxB,CAAC;EAED,MAAM,iCAAqB,gBAAgB,OAAO;AAClD,SAAO;CACR;CAED,AAAQ,aAAaW,KAAqB;AACxC,SAAO,IAAI,QAAQ,OAAO,OAAO,CAAC,QAAQ,MAAM,MAAM;CACvD;;;;;;;;;CAUD,AAAQ,iBACNV,OACAC,GACAU,QACQ;EACR,MAAM,QAAQ;EACd,MAAM,WAAW,SAAS,CAAC,SAAS,EAAE,OAAO,UAAU,GAAG;EAC1D,MAAM,eAAe,MAAM,IAAI,MAAM,IAAI,CAAC,KAAK,KAAK;EAEpD,MAAMC,iBAA2B,CAAE;AACnC,MAAI,KAAK,iBACP,MAAK,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,QAAQ,KAAK,iBAAiB,EAC9D,eAAe,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC;EAIlD,MAAM,iBAAiB,CAAC;aACf,EAAE,KAAK,UAAU,SAAS,cAAc,EAC7C,KAAK,UAAU,SAChB;WACI,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;MACnC,EAAE,SAAS;0BACS,EAClB,KAAK,UAAU,UAChB,GAAG,EAAE,aAAa,WAAW,EAAE,MAAM;YAChC,EAAE,EAAE,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACxC,CAAC;EAGD,MAAM,iCAAqB,gBAAgB,MAAM;AACjD,SAAO;CACR;AACF"}