@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 8.59 kB
Source Map (JSON)
{"version":3,"file":"googlevertexai.cjs","names":["Embeddings","fields?: GoogleVertexAIMultimodalEmbeddingsParams","GoogleVertexAILLMConnection","GoogleAuth","media: GoogleVertexAIMedia","ret: GoogleVertexAIMultimodalEmbeddingsInstance","response: GoogleVertexAILLMResponse<GoogleVertexAIMultimodalEmbeddingsResults>","media: GoogleVertexAIMedia[]","instance: GoogleVertexAIMultimodalEmbeddingsInstance","images: Buffer[]","image: Buffer","documents: string[]","document: string"],"sources":["../../../src/experimental/multimodal_embeddings/googlevertexai.ts"],"sourcesContent":["import { GoogleAuth, GoogleAuthOptions } from \"google-auth-library\";\nimport { Embeddings, EmbeddingsParams } from \"@langchain/core/embeddings\";\nimport { AsyncCallerCallOptions } from \"@langchain/core/utils/async_caller\";\nimport {\n GoogleVertexAIBaseLLMInput,\n GoogleVertexAIBasePrediction,\n GoogleVertexAILLMPredictions,\n} from \"../../types/googlevertexai-types.js\";\nimport {\n GoogleVertexAILLMConnection,\n GoogleVertexAILLMResponse,\n} from \"../../utils/googlevertexai-connection.js\";\n\n/**\n * Parameters for the GoogleVertexAIMultimodalEmbeddings class, extending\n * both EmbeddingsParams and GoogleVertexAIConnectionParams.\n */\nexport interface GoogleVertexAIMultimodalEmbeddingsParams\n extends EmbeddingsParams,\n GoogleVertexAIBaseLLMInput<GoogleAuthOptions> {}\n\n/**\n * Options for the GoogleVertexAIMultimodalEmbeddings class, extending\n * AsyncCallerCallOptions.\n */\ninterface GoogleVertexAIMultimodalEmbeddingsOptions\n extends AsyncCallerCallOptions {}\n\n/**\n * An instance of media (text or image) that can be used for generating\n * embeddings.\n */\ninterface GoogleVertexAIMultimodalEmbeddingsInstance {\n text?: string;\n image?: {\n bytesBase64Encoded: string;\n };\n}\n\n/**\n * The results of generating embeddings, extending\n * GoogleVertexAIBasePrediction. It includes text and image embeddings.\n */\ninterface GoogleVertexAIMultimodalEmbeddingsResults\n extends GoogleVertexAIBasePrediction {\n textEmbedding?: number[];\n imageEmbedding?: number[];\n}\n\n/**\n * The media should have a text property, an image property, or both.\n */\nexport type GoogleVertexAIMedia =\n | {\n text: string;\n image?: Buffer;\n }\n | {\n text?: string;\n image: Buffer;\n };\n\nexport type MediaEmbeddings = {\n text?: number[];\n image?: number[];\n};\n\n/**\n * Class for generating embeddings for text and images using Google's\n * Vertex AI. It extends the Embeddings base class and implements the\n * GoogleVertexAIMultimodalEmbeddingsParams interface.\n */\nexport class GoogleVertexAIMultimodalEmbeddings\n extends Embeddings\n implements GoogleVertexAIMultimodalEmbeddingsParams\n{\n model = \"multimodalembedding@001\";\n\n private connection: GoogleVertexAILLMConnection<\n GoogleVertexAIMultimodalEmbeddingsOptions,\n GoogleVertexAIMultimodalEmbeddingsInstance,\n GoogleVertexAIMultimodalEmbeddingsResults,\n GoogleAuthOptions\n >;\n\n constructor(fields?: GoogleVertexAIMultimodalEmbeddingsParams) {\n super(fields ?? {});\n\n this.model = fields?.model ?? this.model;\n\n this.connection = new GoogleVertexAILLMConnection(\n { ...fields, ...this },\n this.caller,\n new GoogleAuth({\n scopes: \"https://www.googleapis.com/auth/cloud-platform\",\n ...fields?.authOptions,\n })\n );\n }\n\n /**\n * Converts media (text or image) to an instance that can be used for\n * generating embeddings.\n * @param media The media (text or image) to be converted.\n * @returns An instance of media that can be used for generating embeddings.\n */\n mediaToInstance(\n media: GoogleVertexAIMedia\n ): GoogleVertexAIMultimodalEmbeddingsInstance {\n const ret: GoogleVertexAIMultimodalEmbeddingsInstance = {};\n\n if (media?.text) {\n ret.text = media.text;\n }\n\n if (media.image) {\n ret.image = {\n bytesBase64Encoded: media.image.toString(\"base64\"),\n };\n }\n\n return ret;\n }\n\n /**\n * Converts the response from Google Vertex AI to embeddings.\n * @param response The response from Google Vertex AI.\n * @returns An array of media embeddings.\n */\n responseToEmbeddings(\n response: GoogleVertexAILLMResponse<GoogleVertexAIMultimodalEmbeddingsResults>\n ): MediaEmbeddings[] {\n return (\n response?.data as GoogleVertexAILLMPredictions<GoogleVertexAIMultimodalEmbeddingsResults>\n ).predictions.map((r) => ({\n text: r.textEmbedding,\n image: r.imageEmbedding,\n }));\n }\n\n /**\n * Generates embeddings for multiple media instances.\n * @param media An array of media instances.\n * @returns A promise that resolves to an array of media embeddings.\n */\n async embedMedia(media: GoogleVertexAIMedia[]): Promise<MediaEmbeddings[]> {\n // Only one media embedding request is allowed\n return Promise.all(media.map((m) => this.embedMediaQuery(m)));\n }\n\n /**\n * Generates embeddings for a single media instance.\n * @param media A single media instance.\n * @returns A promise that resolves to a media embedding.\n */\n async embedMediaQuery(media: GoogleVertexAIMedia): Promise<MediaEmbeddings> {\n const instance: GoogleVertexAIMultimodalEmbeddingsInstance =\n this.mediaToInstance(media);\n const instances = [instance];\n\n const parameters = {};\n const options = {};\n const responses = await this.connection.request(\n instances,\n parameters,\n options\n );\n\n const result = this.responseToEmbeddings(responses);\n return result[0];\n }\n\n /**\n * Generates embeddings for multiple images.\n * @param images An array of images.\n * @returns A promise that resolves to an array of image embeddings.\n */\n async embedImage(images: Buffer[]): Promise<number[][]> {\n return this.embedMedia(images.map((image) => ({ image }))).then(\n (embeddings) => embeddings.map((e) => e.image ?? [])\n );\n }\n\n /**\n * Generates embeddings for a single image.\n * @param image A single image.\n * @returns A promise that resolves to an image embedding.\n */\n async embedImageQuery(image: Buffer): Promise<number[]> {\n return this.embedMediaQuery({\n image,\n }).then((embeddings) => embeddings.image ?? []);\n }\n\n /**\n * Generates embeddings for multiple text documents.\n * @param documents An array of text documents.\n * @returns A promise that resolves to an array of text document embeddings.\n */\n async embedDocuments(documents: string[]): Promise<number[][]> {\n return this.embedMedia(documents.map((text) => ({ text }))).then(\n (embeddings) => embeddings.map((e) => e.text ?? [])\n );\n }\n\n /**\n * Generates embeddings for a single text document.\n * @param document A single text document.\n * @returns A promise that resolves to a text document embedding.\n */\n async embedQuery(document: string): Promise<number[]> {\n return this.embedMediaQuery({\n text: document,\n }).then((embeddings) => embeddings.text ?? []);\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAwEA,IAAa,qCAAb,cACUA,uCAEV;CACE,QAAQ;CAER,AAAQ;CAOR,YAAYC,QAAmD;EAC7D,MAAM,UAAU,CAAE,EAAC;EAEnB,KAAK,QAAQ,QAAQ,SAAS,KAAK;EAEnC,KAAK,aAAa,IAAIC,8DACpB;GAAE,GAAG;GAAQ,GAAG;EAAM,GACtB,KAAK,QACL,IAAIC,+BAAW;GACb,QAAQ;GACR,GAAG,QAAQ;EACZ;CAEJ;;;;;;;CAQD,gBACEC,OAC4C;EAC5C,MAAMC,MAAkD,CAAE;AAE1D,MAAI,OAAO,MACT,IAAI,OAAO,MAAM;AAGnB,MAAI,MAAM,OACR,IAAI,QAAQ,EACV,oBAAoB,MAAM,MAAM,SAAS,SAAS,CACnD;AAGH,SAAO;CACR;;;;;;CAOD,qBACEC,UACmB;AACnB,UACE,UAAU,MACV,YAAY,IAAI,CAAC,OAAO;GACxB,MAAM,EAAE;GACR,OAAO,EAAE;EACV,GAAE;CACJ;;;;;;CAOD,MAAM,WAAWC,OAA0D;AAEzE,SAAO,QAAQ,IAAI,MAAM,IAAI,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC,CAAC;CAC9D;;;;;;CAOD,MAAM,gBAAgBH,OAAsD;EAC1E,MAAMI,WACJ,KAAK,gBAAgB,MAAM;EAC7B,MAAM,YAAY,CAAC,QAAS;EAE5B,MAAM,aAAa,CAAE;EACrB,MAAM,UAAU,CAAE;EAClB,MAAM,YAAY,MAAM,KAAK,WAAW,QACtC,WACA,YACA,QACD;EAED,MAAM,SAAS,KAAK,qBAAqB,UAAU;AACnD,SAAO,OAAO;CACf;;;;;;CAOD,MAAM,WAAWC,QAAuC;AACtD,SAAO,KAAK,WAAW,OAAO,IAAI,CAAC,WAAW,EAAE,MAAO,GAAE,CAAC,CAAC,KACzD,CAAC,eAAe,WAAW,IAAI,CAAC,MAAM,EAAE,SAAS,CAAE,EAAC,CACrD;CACF;;;;;;CAOD,MAAM,gBAAgBC,OAAkC;AACtD,SAAO,KAAK,gBAAgB,EAC1B,MACD,EAAC,CAAC,KAAK,CAAC,eAAe,WAAW,SAAS,CAAE,EAAC;CAChD;;;;;;CAOD,MAAM,eAAeC,WAA0C;AAC7D,SAAO,KAAK,WAAW,UAAU,IAAI,CAAC,UAAU,EAAE,KAAM,GAAE,CAAC,CAAC,KAC1D,CAAC,eAAe,WAAW,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAE,EAAC,CACpD;CACF;;;;;;CAOD,MAAM,WAAWC,UAAqC;AACpD,SAAO,KAAK,gBAAgB,EAC1B,MAAM,SACP,EAAC,CAAC,KAAK,CAAC,eAAe,WAAW,QAAQ,CAAE,EAAC;CAC/C;AACF"}