@langchain/community/dist/vectorstores/hnswlib.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"hnswlib.cjs","names":["SaveableVectorStore","embeddings: EmbeddingsInterface","args: HNSWLibArgs","documents: Document[]","args: HNSWLibBase","vectors: number[][]","index: HierarchicalNSWT","toSave: Record<string, Document>","query: number[]","k: number","filter?: this[\"FilterType\"]","label: number","params: { directory: string }","directory: string","texts: string[]","metadatas: object[] | object","dbConfig?: {\n      docstore?: SynchronousInMemoryDocstore;\n    }","docs: Document[]","Document","err: any"],"sources":["../../src/vectorstores/hnswlib.ts"],"sourcesContent":["import type {\n  HierarchicalNSW as HierarchicalNSWT,\n  SpaceName,\n} from \"hnswlib-node\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { SaveableVectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\nimport { SynchronousInMemoryDocstore } from \"../stores/doc/in_memory.js\";\n\n/**\n * Interface for the base configuration of HNSWLib. It includes the space\n * name and the number of dimensions.\n */\nexport interface HNSWLibBase {\n  space: SpaceName;\n  numDimensions?: number;\n}\n\n/**\n * Interface for the arguments that can be passed to the HNSWLib\n * constructor. It extends HNSWLibBase and includes properties for the\n * document store and HNSW index.\n */\nexport interface HNSWLibArgs extends HNSWLibBase {\n  docstore?: SynchronousInMemoryDocstore;\n  index?: HierarchicalNSWT;\n}\n\n/**\n * Class that implements a vector store using Hierarchical Navigable Small\n * World (HNSW) graphs. It extends the SaveableVectorStore class and\n * provides methods for adding documents and vectors, performing\n * similarity searches, and saving and loading the vector store.\n */\nexport class HNSWLib extends SaveableVectorStore {\n  declare FilterType: (doc: Document) => boolean;\n\n  _index?: HierarchicalNSWT;\n\n  docstore: SynchronousInMemoryDocstore;\n\n  args: HNSWLibBase;\n\n  _vectorstoreType(): string {\n    return \"hnswlib\";\n  }\n\n  constructor(embeddings: EmbeddingsInterface, args: HNSWLibArgs) {\n    super(embeddings, args);\n    this._index = args.index;\n    this.args = args;\n    this.embeddings = embeddings;\n    this.docstore = args?.docstore ?? new SynchronousInMemoryDocstore();\n  }\n\n  /**\n   * Method to add documents to the vector store. It first converts the\n   * documents to vectors using the embeddings, then adds the vectors to the\n   * vector store.\n   * @param documents The documents to be added to the vector store.\n   * @returns A Promise that resolves when the documents have been added.\n   */\n  async addDocuments(documents: Document[]): Promise<void> {\n    const texts = documents.map(({ pageContent }) => pageContent);\n    return this.addVectors(\n      await this.embeddings.embedDocuments(texts),\n      documents\n    );\n  }\n\n  private static async getHierarchicalNSW(args: HNSWLibBase) {\n    const { HierarchicalNSW } = await HNSWLib.imports();\n    if (!args.space) {\n      throw new Error(\"hnswlib-node requires a space argument\");\n    }\n    if (args.numDimensions === undefined) {\n      throw new Error(\"hnswlib-node requires a numDimensions argument\");\n    }\n    return new HierarchicalNSW(args.space, args.numDimensions);\n  }\n\n  private async initIndex(vectors: number[][]) {\n    if (!this._index) {\n      if (this.args.numDimensions === undefined) {\n        this.args.numDimensions = vectors[0].length;\n      }\n      this.index = await HNSWLib.getHierarchicalNSW(this.args);\n    }\n    if (!this.index.getCurrentCount()) {\n      this.index.initIndex(vectors.length);\n    }\n  }\n\n  public get index(): HierarchicalNSWT {\n    if (!this._index) {\n      throw new Error(\n        \"Vector store not initialised yet. Try calling `addTexts` first.\"\n      );\n    }\n    return this._index;\n  }\n\n  private set index(index: HierarchicalNSWT) {\n    this._index = index;\n  }\n\n  /**\n   * Method to add vectors to the vector store. It first initializes the\n   * index if it hasn't been initialized yet, then adds the vectors to the\n   * index and the documents to the document store.\n   * @param vectors The vectors to be added to the vector store.\n   * @param documents The documents corresponding to the vectors.\n   * @returns A Promise that resolves when the vectors and documents have been added.\n   */\n  async addVectors(vectors: number[][], documents: Document[]) {\n    if (vectors.length === 0) {\n      return;\n    }\n    await this.initIndex(vectors);\n\n    // TODO here we could optionally normalise the vectors to unit length\n    // so that dot product is equivalent to cosine similarity, like this\n    // https://github.com/nmslib/hnswlib/issues/384#issuecomment-1155737730\n    // While we only support OpenAI embeddings this isn't necessary\n    if (vectors.length !== documents.length) {\n      throw new Error(`Vectors and metadatas must have the same length`);\n    }\n    if (vectors[0].length !== this.args.numDimensions) {\n      throw new Error(\n        `Vectors must have the same length as the number of dimensions (${this.args.numDimensions})`\n      );\n    }\n    const capacity = this.index.getMaxElements();\n    const needed = this.index.getCurrentCount() + vectors.length;\n    if (needed > capacity) {\n      this.index.resizeIndex(needed);\n    }\n    const docstoreSize = this.index.getCurrentCount();\n    const toSave: Record<string, Document> = {};\n    for (let i = 0; i < vectors.length; i += 1) {\n      this.index.addPoint(vectors[i], docstoreSize + i);\n      toSave[docstoreSize + i] = documents[i];\n    }\n    this.docstore.add(toSave);\n  }\n\n  /**\n   * Method to perform a similarity search in the vector store using a query\n   * vector. It returns the k most similar documents along with their\n   * similarity scores. An optional filter function can be provided to\n   * filter the documents.\n   * @param query The query vector.\n   * @param k The number of most similar documents to return.\n   * @param filter An optional filter function to filter the documents.\n   * @returns A Promise that resolves to an array of tuples, where each tuple contains a document and its similarity score.\n   */\n  async similaritySearchVectorWithScore(\n    query: number[],\n    k: number,\n    filter?: this[\"FilterType\"]\n  ) {\n    if (this.args.numDimensions && !this._index) {\n      await this.initIndex([[]]);\n    }\n    if (query.length !== this.args.numDimensions) {\n      throw new Error(\n        `Query vector must have the same length as the number of dimensions (${this.args.numDimensions})`\n      );\n    }\n    if (k > this.index.getCurrentCount()) {\n      const total = this.index.getCurrentCount();\n      console.warn(\n        `k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`\n      );\n      // eslint-disable-next-line no-param-reassign\n      k = total;\n    }\n    const filterFunction = (label: number): boolean => {\n      if (!filter) {\n        return true;\n      }\n      const document = this.docstore.search(String(label));\n      if (typeof document !== \"string\") {\n        return filter(document);\n      }\n      return false;\n    };\n    const result = this.index.searchKnn(\n      query,\n      k,\n      filter ? filterFunction : undefined\n    );\n    return result.neighbors.map(\n      (docIndex, resultIndex) =>\n        [\n          this.docstore.search(String(docIndex)),\n          result.distances[resultIndex],\n        ] as [Document, number]\n    );\n  }\n\n  /**\n   * Method to delete the vector store from a directory. It deletes the\n   * hnswlib.index file, the docstore.json file, and the args.json file from\n   * the directory.\n   * @param params An object with a directory property that specifies the directory from which to delete the vector store.\n   * @returns A Promise that resolves when the vector store has been deleted.\n   */\n  async delete(params: { directory: string }) {\n    const fs = await import(\"node:fs/promises\");\n    const path = await import(\"node:path\");\n    try {\n      await fs.access(path.join(params.directory, \"hnswlib.index\"));\n    } catch {\n      throw new Error(\n        `Directory ${params.directory} does not contain a hnswlib.index file.`\n      );\n    }\n\n    await Promise.all([\n      await fs.rm(path.join(params.directory, \"hnswlib.index\"), {\n        force: true,\n      }),\n      await fs.rm(path.join(params.directory, \"docstore.json\"), {\n        force: true,\n      }),\n      await fs.rm(path.join(params.directory, \"args.json\"), { force: true }),\n    ]);\n  }\n\n  /**\n   * Method to save the vector store to a directory. It saves the HNSW\n   * index, the arguments, and the document store to the directory.\n   * @param directory The directory to which to save the vector store.\n   * @returns A Promise that resolves when the vector store has been saved.\n   */\n  async save(directory: string) {\n    const fs = await import(\"node:fs/promises\");\n    const path = await import(\"node:path\");\n    await fs.mkdir(directory, { recursive: true });\n    await Promise.all([\n      this.index.writeIndex(path.join(directory, \"hnswlib.index\")),\n      await fs.writeFile(\n        path.join(directory, \"args.json\"),\n        JSON.stringify(this.args)\n      ),\n      await fs.writeFile(\n        path.join(directory, \"docstore.json\"),\n        JSON.stringify(Array.from(this.docstore._docs.entries()))\n      ),\n    ]);\n  }\n\n  /**\n   * Static method to load a vector store from a directory. It reads the\n   * HNSW index, the arguments, and the document store from the directory,\n   * then creates a new HNSWLib instance with these values.\n   * @param directory The directory from which to load the vector store.\n   * @param embeddings The embeddings to be used by the HNSWLib instance.\n   * @returns A Promise that resolves to a new HNSWLib instance.\n   */\n  static async load(directory: string, embeddings: EmbeddingsInterface) {\n    const fs = await import(\"node:fs/promises\");\n    const path = await import(\"node:path\");\n    const args = JSON.parse(\n      await fs.readFile(path.join(directory, \"args.json\"), \"utf8\")\n    );\n    const index = await HNSWLib.getHierarchicalNSW(args);\n    const [docstoreFiles] = await Promise.all([\n      fs\n        .readFile(path.join(directory, \"docstore.json\"), \"utf8\")\n        .then(JSON.parse),\n      index.readIndex(path.join(directory, \"hnswlib.index\")),\n    ]);\n    args.docstore = new SynchronousInMemoryDocstore(new Map(docstoreFiles));\n\n    args.index = index;\n\n    return new HNSWLib(embeddings, args);\n  }\n\n  /**\n   * Static method to create a new HNSWLib instance from texts and metadata.\n   * It creates a new Document instance for each text and metadata, then\n   * calls the fromDocuments method to create the HNSWLib instance.\n   * @param texts The texts to be used to create the documents.\n   * @param metadatas The metadata to be used to create the documents.\n   * @param embeddings The embeddings to be used by the HNSWLib instance.\n   * @param dbConfig An optional configuration object for the document store.\n   * @returns A Promise that resolves to a new HNSWLib instance.\n   */\n  static async fromTexts(\n    texts: string[],\n    metadatas: object[] | object,\n    embeddings: EmbeddingsInterface,\n    dbConfig?: {\n      docstore?: SynchronousInMemoryDocstore;\n    }\n  ): Promise<HNSWLib> {\n    const docs: Document[] = [];\n    for (let i = 0; i < texts.length; i += 1) {\n      const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n      const newDoc = new Document({\n        pageContent: texts[i],\n        metadata,\n      });\n      docs.push(newDoc);\n    }\n    return HNSWLib.fromDocuments(docs, embeddings, dbConfig);\n  }\n\n  /**\n   * Static method to create a new HNSWLib instance from documents. It\n   * creates a new HNSWLib instance, adds the documents to it, then returns\n   * the instance.\n   * @param docs The documents to be added to the HNSWLib instance.\n   * @param embeddings The embeddings to be used by the HNSWLib instance.\n   * @param dbConfig An optional configuration object for the document store.\n   * @returns A Promise that resolves to a new HNSWLib instance.\n   */\n  static async fromDocuments(\n    docs: Document[],\n    embeddings: EmbeddingsInterface,\n    dbConfig?: {\n      docstore?: SynchronousInMemoryDocstore;\n    }\n  ): Promise<HNSWLib> {\n    const args: HNSWLibArgs = {\n      docstore: dbConfig?.docstore,\n      space: \"cosine\",\n    };\n    const instance = new this(embeddings, args);\n    await instance.addDocuments(docs);\n    return instance;\n  }\n\n  static async imports(): Promise<{\n    HierarchicalNSW: typeof HierarchicalNSWT;\n  }> {\n    try {\n      const {\n        default: { HierarchicalNSW },\n      } = await import(\"hnswlib-node\");\n\n      return { HierarchicalNSW };\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    } catch (err: any) {\n      throw new Error(\n        `Could not import hnswlib-node. Please install hnswlib-node as a dependency with, e.g. \\`npm install -S hnswlib-node\\`.\\n\\nError: ${err?.message}`\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAkCA,IAAa,UAAb,MAAa,gBAAgBA,kDAAoB;CAG/C;CAEA;CAEA;CAEA,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAmB;EAC9D,MAAM,YAAY,KAAK;EACvB,KAAK,SAAS,KAAK;EACnB,KAAK,OAAO;EACZ,KAAK,aAAa;EAClB,KAAK,WAAW,MAAM,YAAY,mDAAI;CACvC;;;;;;;;CASD,MAAM,aAAaC,WAAsC;EACvD,MAAM,QAAQ,UAAU,IAAI,CAAC,EAAE,aAAa,KAAK,YAAY;AAC7D,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,MAAM,EAC3C,UACD;CACF;CAED,aAAqB,mBAAmBC,MAAmB;EACzD,MAAM,EAAE,iBAAiB,GAAG,MAAM,QAAQ,SAAS;AACnD,MAAI,CAAC,KAAK,MACR,OAAM,IAAI,MAAM;AAElB,MAAI,KAAK,kBAAkB,OACzB,OAAM,IAAI,MAAM;AAElB,SAAO,IAAI,gBAAgB,KAAK,OAAO,KAAK;CAC7C;CAED,MAAc,UAAUC,SAAqB;AAC3C,MAAI,CAAC,KAAK,QAAQ;AAChB,OAAI,KAAK,KAAK,kBAAkB,QAC9B,KAAK,KAAK,gBAAgB,QAAQ,GAAG;GAEvC,KAAK,QAAQ,MAAM,QAAQ,mBAAmB,KAAK,KAAK;EACzD;AACD,MAAI,CAAC,KAAK,MAAM,iBAAiB,EAC/B,KAAK,MAAM,UAAU,QAAQ,OAAO;CAEvC;CAED,IAAW,QAA0B;AACnC,MAAI,CAAC,KAAK,OACR,OAAM,IAAI,MACR;AAGJ,SAAO,KAAK;CACb;CAED,IAAY,MAAMC,OAAyB;EACzC,KAAK,SAAS;CACf;;;;;;;;;CAUD,MAAM,WAAWD,SAAqBF,WAAuB;AAC3D,MAAI,QAAQ,WAAW,EACrB;EAEF,MAAM,KAAK,UAAU,QAAQ;AAM7B,MAAI,QAAQ,WAAW,UAAU,OAC/B,OAAM,IAAI,MAAM,CAAC,+CAA+C,CAAC;AAEnE,MAAI,QAAQ,GAAG,WAAW,KAAK,KAAK,cAClC,OAAM,IAAI,MACR,CAAC,+DAA+D,EAAE,KAAK,KAAK,cAAc,CAAC,CAAC;EAGhG,MAAM,WAAW,KAAK,MAAM,gBAAgB;EAC5C,MAAM,SAAS,KAAK,MAAM,iBAAiB,GAAG,QAAQ;AACtD,MAAI,SAAS,UACX,KAAK,MAAM,YAAY,OAAO;EAEhC,MAAM,eAAe,KAAK,MAAM,iBAAiB;EACjD,MAAMI,SAAmC,CAAE;AAC3C,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GAAG;GAC1C,KAAK,MAAM,SAAS,QAAQ,IAAI,eAAe,EAAE;GACjD,OAAO,eAAe,KAAK,UAAU;EACtC;EACD,KAAK,SAAS,IAAI,OAAO;CAC1B;;;;;;;;;;;CAYD,MAAM,gCACJC,OACAC,GACAC,QACA;AACA,MAAI,KAAK,KAAK,iBAAiB,CAAC,KAAK,QACnC,MAAM,KAAK,UAAU,CAAC,CAAE,CAAC,EAAC;AAE5B,MAAI,MAAM,WAAW,KAAK,KAAK,cAC7B,OAAM,IAAI,MACR,CAAC,oEAAoE,EAAE,KAAK,KAAK,cAAc,CAAC,CAAC;AAGrG,MAAI,IAAI,KAAK,MAAM,iBAAiB,EAAE;GACpC,MAAM,QAAQ,KAAK,MAAM,iBAAiB;GAC1C,QAAQ,KACN,CAAC,GAAG,EAAE,EAAE,uDAAuD,EAAE,MAAM,gBAAgB,EAAE,OAAO,CACjG;GAED,IAAI;EACL;EACD,MAAM,iBAAiB,CAACC,UAA2B;AACjD,OAAI,CAAC,OACH,QAAO;GAET,MAAM,WAAW,KAAK,SAAS,OAAO,OAAO,MAAM,CAAC;AACpD,OAAI,OAAO,aAAa,SACtB,QAAO,OAAO,SAAS;AAEzB,UAAO;EACR;EACD,MAAM,SAAS,KAAK,MAAM,UACxB,OACA,GACA,SAAS,iBAAiB,OAC3B;AACD,SAAO,OAAO,UAAU,IACtB,CAAC,UAAU,gBACT,CACE,KAAK,SAAS,OAAO,OAAO,SAAS,CAAC,EACtC,OAAO,UAAU,YAClB,EACJ;CACF;;;;;;;;CASD,MAAM,OAAOC,QAA+B;EAC1C,MAAM,KAAK,MAAM,OAAO;EACxB,MAAM,OAAO,MAAM,OAAO;AAC1B,MAAI;GACF,MAAM,GAAG,OAAO,KAAK,KAAK,OAAO,WAAW,gBAAgB,CAAC;EAC9D,QAAO;AACN,SAAM,IAAI,MACR,CAAC,UAAU,EAAE,OAAO,UAAU,uCAAuC,CAAC;EAEzE;EAED,MAAM,QAAQ,IAAI;GAChB,MAAM,GAAG,GAAG,KAAK,KAAK,OAAO,WAAW,gBAAgB,EAAE,EACxD,OAAO,KACR,EAAC;GACF,MAAM,GAAG,GAAG,KAAK,KAAK,OAAO,WAAW,gBAAgB,EAAE,EACxD,OAAO,KACR,EAAC;GACF,MAAM,GAAG,GAAG,KAAK,KAAK,OAAO,WAAW,YAAY,EAAE,EAAE,OAAO,KAAM,EAAC;EACvE,EAAC;CACH;;;;;;;CAQD,MAAM,KAAKC,WAAmB;EAC5B,MAAM,KAAK,MAAM,OAAO;EACxB,MAAM,OAAO,MAAM,OAAO;EAC1B,MAAM,GAAG,MAAM,WAAW,EAAE,WAAW,KAAM,EAAC;EAC9C,MAAM,QAAQ,IAAI;GAChB,KAAK,MAAM,WAAW,KAAK,KAAK,WAAW,gBAAgB,CAAC;GAC5D,MAAM,GAAG,UACP,KAAK,KAAK,WAAW,YAAY,EACjC,KAAK,UAAU,KAAK,KAAK,CAC1B;GACD,MAAM,GAAG,UACP,KAAK,KAAK,WAAW,gBAAgB,EACrC,KAAK,UAAU,MAAM,KAAK,KAAK,SAAS,MAAM,SAAS,CAAC,CAAC,CAC1D;EACF,EAAC;CACH;;;;;;;;;CAUD,aAAa,KAAKA,WAAmBZ,YAAiC;EACpE,MAAM,KAAK,MAAM,OAAO;EACxB,MAAM,OAAO,MAAM,OAAO;EAC1B,MAAM,OAAO,KAAK,MAChB,MAAM,GAAG,SAAS,KAAK,KAAK,WAAW,YAAY,EAAE,OAAO,CAC7D;EACD,MAAM,QAAQ,MAAM,QAAQ,mBAAmB,KAAK;EACpD,MAAM,CAAC,cAAc,GAAG,MAAM,QAAQ,IAAI,CACxC,GACG,SAAS,KAAK,KAAK,WAAW,gBAAgB,EAAE,OAAO,CACvD,KAAK,KAAK,MAAM,EACnB,MAAM,UAAU,KAAK,KAAK,WAAW,gBAAgB,CAAC,AACvD,EAAC;EACF,KAAK,WAAW,mDAAI,4BAA4B,IAAI,IAAI;EAExD,KAAK,QAAQ;AAEb,SAAO,IAAI,QAAQ,YAAY;CAChC;;;;;;;;;;;CAYD,aAAa,UACXa,OACAC,WACAd,YACAe,UAGkB;EAClB,MAAMC,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,SAAS,IAAIC,oCAAS;IAC1B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,OAAO;EAClB;AACD,SAAO,QAAQ,cAAc,MAAM,YAAY,SAAS;CACzD;;;;;;;;;;CAWD,aAAa,cACXD,MACAhB,YACAe,UAGkB;EAClB,MAAMd,OAAoB;GACxB,UAAU,UAAU;GACpB,OAAO;EACR;EACD,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;CAED,aAAa,UAEV;AACD,MAAI;GACF,MAAM,EACJ,SAAS,EAAE,iBAAiB,EAC7B,GAAG,MAAM,OAAO;AAEjB,UAAO,EAAE,gBAAiB;EAE3B,SAAQiB,KAAU;AACjB,SAAM,IAAI,MACR,CAAC,iIAAiI,EAAE,KAAK,SAAS;EAErJ;CACF;AACF"}