@langchain/community/dist/vectorstores/turbopuffer.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"turbopuffer.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: TurbopufferParams","AsyncCaller","fetchUrl: string","stringifiedBody: string | undefined","headers: Record<string, string>","vectors: number[][]","documents: DocumentInterface[]","options?: { ids?: string[] }","batchedVectors: number[][][]","batchedDocuments: DocumentInterface[][]","attributes: Record<string, (string | null)[]>","params: { deleteIndex?: boolean }","query: number[]","k: number","includeVector?: boolean","filter?: this[\"FilterType\"]","result: [DocumentInterface, number][]","Document","docs: DocumentInterface[]","dbConfig: TurbopufferParams"],"sources":["../../src/vectorstores/turbopuffer.ts"],"sourcesContent":["import { v4 as uuidv4 } from \"uuid\";\nimport { type DocumentInterface, Document } from \"@langchain/core/documents\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport {\n  AsyncCaller,\n  AsyncCallerParams,\n} from \"@langchain/core/utils/async_caller\";\nimport { chunkArray } from \"@langchain/core/utils/chunk_array\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\n\n/**\n * @deprecated Use `TurbopufferDistanceMetric` from `@langchain/turbopuffer` instead.\n */\nexport type TurbopufferDistanceMetric = \"cosine_distance\" | \"euclidean_squared\";\n\n/**\n * @deprecated Use `TurbopufferFilter` from `@langchain/turbopuffer` instead.\n */\nexport type TurbopufferFilterType = Record<\n  string,\n  Array<[string, string[] | string]>\n>;\n\n/**\n * @deprecated Use `TurbopufferParams` from `@langchain/turbopuffer` instead.\n */\nexport interface TurbopufferParams extends AsyncCallerParams {\n  apiKey?: string;\n  namespace?: string;\n  distanceMetric?: TurbopufferDistanceMetric;\n  apiUrl?: string;\n  batchSize?: number;\n}\n\n/**\n * @deprecated Use `@langchain/turbopuffer` instead.\n */\nexport interface TurbopufferQueryResult {\n  dist: number;\n  id: number;\n  vector?: number[];\n  attributes: Record<string, string>;\n}\n\n/**\n * @deprecated Use `TurbopufferVectorStore` from `@langchain/turbopuffer` instead.\n * The new package uses the official SDK and supports document deletion, region\n * configuration, and returns document IDs in search results.\n */\nexport class TurbopufferVectorStore extends VectorStore {\n  declare FilterType: TurbopufferFilterType;\n\n  get lc_secrets(): { [key: string]: string } {\n    return {\n      apiKey: \"TURBOPUFFER_API_KEY\",\n    };\n  }\n\n  get lc_aliases(): { [key: string]: string } {\n    return {\n      apiKey: \"TURBOPUFFER_API_KEY\",\n    };\n  }\n\n  // Handle minification for tracing\n  static lc_name(): string {\n    return \"TurbopufferVectorStore\";\n  }\n\n  protected distanceMetric: TurbopufferDistanceMetric = \"cosine_distance\";\n\n  protected apiKey: string;\n\n  protected namespace = \"default\";\n\n  protected apiUrl = \"https://api.turbopuffer.com/v1\";\n\n  caller: AsyncCaller;\n\n  batchSize = 3000;\n\n  public _vectorstoreType(): string {\n    return \"turbopuffer\";\n  }\n\n  constructor(embeddings: EmbeddingsInterface, args: TurbopufferParams) {\n    super(embeddings, args);\n\n    const {\n      apiKey: argsApiKey,\n      namespace,\n      distanceMetric,\n      apiUrl,\n      batchSize,\n      ...asyncCallerArgs\n    } = args;\n\n    const apiKey = argsApiKey ?? getEnvironmentVariable(\"TURBOPUFFER_API_KEY\");\n    if (!apiKey) {\n      throw new Error(\n        `Turbopuffer API key not found.\\nPlease pass it in as \"apiKey\" or set it as an environment variable called \"TURBOPUFFER_API_KEY\"`\n      );\n    }\n    this.apiKey = apiKey;\n    this.namespace = namespace ?? this.namespace;\n    this.distanceMetric = distanceMetric ?? this.distanceMetric;\n    this.apiUrl = apiUrl ?? this.apiUrl;\n    this.batchSize = batchSize ?? this.batchSize;\n    this.caller = new AsyncCaller({\n      maxConcurrency: 6,\n      maxRetries: 0,\n      ...asyncCallerArgs,\n    });\n  }\n\n  defaultHeaders() {\n    return {\n      Authorization: `Bearer ${this.apiKey}`,\n      \"Content-Type\": \"application/json\",\n    };\n  }\n\n  async callWithRetry(\n    fetchUrl: string,\n    stringifiedBody: string | undefined,\n    method = \"POST\"\n  ) {\n    const json = await this.caller.call(async () => {\n      const headers: Record<string, string> = {\n        Authorization: `Bearer ${this.apiKey}`,\n      };\n      if (stringifiedBody !== undefined) {\n        headers[\"Content-Type\"] = \"application/json\";\n      }\n      const response = await fetch(fetchUrl, {\n        method,\n        headers,\n        body: stringifiedBody,\n      });\n      if (response.status !== 200) {\n        const error = new Error(\n          `Failed to call turbopuffer. Response status ${\n            response.status\n          }\\nFull response: ${await response.text()}`\n        );\n        // eslint-disable-next-line @typescript-eslint/no-explicit-any\n        (error as any).response = response;\n        throw error;\n      }\n      return response.json();\n    });\n\n    return json;\n  }\n\n  async addVectors(\n    vectors: number[][],\n    documents: DocumentInterface[],\n    options?: { ids?: string[] }\n  ): Promise<string[]> {\n    if (options?.ids && options.ids.length !== vectors.length) {\n      throw new Error(\n        \"Number of ids provided does not match number of vectors\"\n      );\n    }\n\n    if (documents.length !== vectors.length) {\n      throw new Error(\n        \"Number of documents provided does not match number of vectors\"\n      );\n    }\n\n    if (documents.length === 0) {\n      throw new Error(\"No documents provided\");\n    }\n\n    const batchedVectors: number[][][] = chunkArray(vectors, this.batchSize);\n    const batchedDocuments: DocumentInterface[][] = chunkArray(\n      documents,\n      this.batchSize\n    );\n    const batchedIds = options?.ids\n      ? chunkArray(options.ids, this.batchSize)\n      : batchedDocuments.map((docs) => docs.map((_) => uuidv4()));\n\n    const batchRequests = batchedVectors.map(async (batchVectors, index) => {\n      const batchDocs = batchedDocuments[index];\n      const batchIds = batchedIds[index];\n\n      if (batchIds.length !== batchVectors.length) {\n        throw new Error(\n          \"Number of ids provided does not match number of vectors\"\n        );\n      }\n\n      const attributes: Record<string, (string | null)[]> = {\n        __lc_page_content: batchDocs.map((doc) => doc.pageContent),\n      };\n\n      const usedMetadataFields = new Set(\n        batchDocs.map((doc) => Object.keys(doc.metadata)).flat()\n      );\n\n      for (const key of usedMetadataFields) {\n        attributes[key] = batchDocs.map((doc) => {\n          if (doc.metadata[key] !== undefined) {\n            if (typeof doc.metadata[key] === \"string\") {\n              return doc.metadata[key];\n            } else {\n              console.warn(\n                [\n                  `[WARNING]: Dropping non-string metadata key \"${key}\" with value \"${JSON.stringify(\n                    doc.metadata[key]\n                  )}\".`,\n                  `turbopuffer currently supports only string metadata values.`,\n                ].join(\"\\n\")\n              );\n              return null;\n            }\n          } else {\n            return null;\n          }\n        });\n      }\n\n      const data = {\n        ids: batchIds,\n        vectors: batchVectors,\n        distance_metric: this.distanceMetric,\n        attributes,\n      };\n\n      return this.callWithRetry(\n        `${this.apiUrl}/vectors/${this.namespace}`,\n        JSON.stringify(data)\n      );\n    });\n\n    // Execute all batch requests in parallel\n    await Promise.all(batchRequests);\n    return batchedIds.flat();\n  }\n\n  async delete(params: { deleteIndex?: boolean }): Promise<void> {\n    if (params.deleteIndex) {\n      await this.callWithRetry(\n        `${this.apiUrl}/vectors/${this.namespace}`,\n        undefined,\n        \"DELETE\"\n      );\n    } else {\n      throw new Error(`You must provide a \"deleteIndex\" flag.`);\n    }\n  }\n\n  async addDocuments(\n    documents: DocumentInterface[],\n    options?: { ids?: string[] }\n  ): Promise<string[]> {\n    const vectors = await this.embeddings.embedDocuments(\n      documents.map((doc) => doc.pageContent)\n    );\n\n    return this.addVectors(vectors, documents, options);\n  }\n\n  protected async queryVectors(\n    query: number[],\n    k: number,\n    includeVector?: boolean,\n    // See https://Turbopuffer.com/docs/reference/query for more info\n    filter?: this[\"FilterType\"]\n  ): Promise<TurbopufferQueryResult[]> {\n    const data = {\n      vector: query,\n      top_k: k,\n      distance_metric: this.distanceMetric,\n      filters: filter,\n      include_attributes: true,\n      include_vectors: includeVector,\n    };\n\n    return this.callWithRetry(\n      `${this.apiUrl}/vectors/${this.namespace}/query`,\n      JSON.stringify(data)\n    );\n  }\n\n  async similaritySearchVectorWithScore(\n    query: number[],\n    k: number,\n    filter?: this[\"FilterType\"]\n  ): Promise<[DocumentInterface, number][]> {\n    const search = await this.queryVectors(query, k, false, filter);\n    const result: [DocumentInterface, number][] = search.map((res) => {\n      const { __lc_page_content, ...metadata } = res.attributes;\n      return [\n        new Document({\n          pageContent: __lc_page_content,\n          metadata,\n        }),\n        res.dist,\n      ];\n    });\n\n    return result;\n  }\n\n  static async fromDocuments(\n    docs: DocumentInterface[],\n    embeddings: EmbeddingsInterface,\n    dbConfig: TurbopufferParams\n  ): Promise<TurbopufferVectorStore> {\n    const instance = new this(embeddings, dbConfig);\n    await instance.addDocuments(docs);\n    return instance;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkDA,IAAa,yBAAb,cAA4CA,0CAAY;CAGtD,IAAI,aAAwC;AAC1C,SAAO,EACL,QAAQ,sBACT;CACF;CAED,IAAI,aAAwC;AAC1C,SAAO,EACL,QAAQ,sBACT;CACF;CAGD,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,AAAU,iBAA4C;CAEtD,AAAU;CAEV,AAAU,YAAY;CAEtB,AAAU,SAAS;CAEnB;CAEA,YAAY;CAEZ,AAAO,mBAA2B;AAChC,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAyB;EACpE,MAAM,YAAY,KAAK;EAEvB,MAAM,EACJ,QAAQ,YACR,WACA,gBACA,QACA,UACA,GAAG,iBACJ,GAAG;EAEJ,MAAM,SAAS,qEAAqC,sBAAsB;AAC1E,MAAI,CAAC,OACH,OAAM,IAAI,MACR,CAAC,+HAA+H,CAAC;EAGrI,KAAK,SAAS;EACd,KAAK,YAAY,aAAa,KAAK;EACnC,KAAK,iBAAiB,kBAAkB,KAAK;EAC7C,KAAK,SAAS,UAAU,KAAK;EAC7B,KAAK,YAAY,aAAa,KAAK;EACnC,KAAK,SAAS,IAAIC,gDAAY;GAC5B,gBAAgB;GAChB,YAAY;GACZ,GAAG;EACJ;CACF;CAED,iBAAiB;AACf,SAAO;GACL,eAAe,CAAC,OAAO,EAAE,KAAK,QAAQ;GACtC,gBAAgB;EACjB;CACF;CAED,MAAM,cACJC,UACAC,iBACA,SAAS,QACT;EACA,MAAM,OAAO,MAAM,KAAK,OAAO,KAAK,YAAY;GAC9C,MAAMC,UAAkC,EACtC,eAAe,CAAC,OAAO,EAAE,KAAK,QAAQ,CACvC;AACD,OAAI,oBAAoB,QACtB,QAAQ,kBAAkB;GAE5B,MAAM,WAAW,MAAM,MAAM,UAAU;IACrC;IACA;IACA,MAAM;GACP,EAAC;AACF,OAAI,SAAS,WAAW,KAAK;IAC3B,MAAM,wBAAQ,IAAI,MAChB,CAAC,4CAA4C,EAC3C,SAAS,OACV,iBAAiB,EAAE,MAAM,SAAS,MAAM,EAAE;IAG5C,MAAc,WAAW;AAC1B,UAAM;GACP;AACD,UAAO,SAAS,MAAM;EACvB,EAAC;AAEF,SAAO;CACR;CAED,MAAM,WACJC,SACAC,WACAC,SACmB;AACnB,MAAI,SAAS,OAAO,QAAQ,IAAI,WAAW,QAAQ,OACjD,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,WAAW,QAAQ,OAC/B,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,WAAW,EACvB,OAAM,IAAI,MAAM;EAGlB,MAAMC,oEAA0C,SAAS,KAAK,UAAU;EACxE,MAAMC,sEACJ,WACA,KAAK,UACN;EACD,MAAM,aAAa,SAAS,yDACb,QAAQ,KAAK,KAAK,UAAU,GACvC,iBAAiB,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,oBAAc,CAAC,CAAC;EAE7D,MAAM,gBAAgB,eAAe,IAAI,OAAO,cAAc,UAAU;GACtE,MAAM,YAAY,iBAAiB;GACnC,MAAM,WAAW,WAAW;AAE5B,OAAI,SAAS,WAAW,aAAa,OACnC,OAAM,IAAI,MACR;GAIJ,MAAMC,aAAgD,EACpD,mBAAmB,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY,CAC3D;GAED,MAAM,qBAAqB,IAAI,IAC7B,UAAU,IAAI,CAAC,QAAQ,OAAO,KAAK,IAAI,SAAS,CAAC,CAAC,MAAM;AAG1D,QAAK,MAAM,OAAO,oBAChB,WAAW,OAAO,UAAU,IAAI,CAAC,QAAQ;AACvC,QAAI,IAAI,SAAS,SAAS,OACxB,KAAI,OAAO,IAAI,SAAS,SAAS,SAC/B,QAAO,IAAI,SAAS;SACf;KACL,QAAQ,KACN,CACE,CAAC,6CAA6C,EAAE,IAAI,cAAc,EAAE,KAAK,UACvE,IAAI,SAAS,KACd,CAAC,EAAE,CAAC,EACL,CAAC,2DAA2D,CAAC,AAC9D,EAAC,KAAK,KAAK,CACb;AACD,YAAO;IACR;QAED,QAAO;GAEV,EAAC;GAGJ,MAAM,OAAO;IACX,KAAK;IACL,SAAS;IACT,iBAAiB,KAAK;IACtB;GACD;AAED,UAAO,KAAK,cACV,GAAG,KAAK,OAAO,SAAS,EAAE,KAAK,WAAW,EAC1C,KAAK,UAAU,KAAK,CACrB;EACF,EAAC;EAGF,MAAM,QAAQ,IAAI,cAAc;AAChC,SAAO,WAAW,MAAM;CACzB;CAED,MAAM,OAAOC,QAAkD;AAC7D,MAAI,OAAO,aACT,MAAM,KAAK,cACT,GAAG,KAAK,OAAO,SAAS,EAAE,KAAK,WAAW,EAC1C,QACA,SACD;MAED,OAAM,IAAI,MAAM,CAAC,sCAAsC,CAAC;CAE3D;CAED,MAAM,aACJL,WACAC,SACmB;EACnB,MAAM,UAAU,MAAM,KAAK,WAAW,eACpC,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY,CACxC;AAED,SAAO,KAAK,WAAW,SAAS,WAAW,QAAQ;CACpD;CAED,MAAgB,aACdK,OACAC,GACAC,eAEAC,QACmC;EACnC,MAAM,OAAO;GACX,QAAQ;GACR,OAAO;GACP,iBAAiB,KAAK;GACtB,SAAS;GACT,oBAAoB;GACpB,iBAAiB;EAClB;AAED,SAAO,KAAK,cACV,GAAG,KAAK,OAAO,SAAS,EAAE,KAAK,UAAU,MAAM,CAAC,EAChD,KAAK,UAAU,KAAK,CACrB;CACF;CAED,MAAM,gCACJH,OACAC,GACAE,QACwC;EACxC,MAAM,SAAS,MAAM,KAAK,aAAa,OAAO,GAAG,OAAO,OAAO;EAC/D,MAAMC,SAAwC,OAAO,IAAI,CAAC,QAAQ;GAChE,MAAM,EAAE,kBAAmB,GAAG,UAAU,GAAG,IAAI;AAC/C,UAAO,CACL,IAAIC,oCAAS;IACX,aAAa;IACb;GACD,IACD,IAAI,IACL;EACF,EAAC;AAEF,SAAO;CACR;CAED,aAAa,cACXC,MACAnB,YACAoB,UACiC;EACjC,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;AACF"}