@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 4.32 kB
Source Map (JSON)
{"version":3,"file":"google_cloud_storage.cjs","names":["BaseDocumentLoader","fsDefault","path","os","Storage","e: any","UnstructuredLoader"],"sources":["../../../src/document_loaders/web/google_cloud_storage.ts"],"sourcesContent":["import { Storage, StorageOptions } from \"@google-cloud/storage\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as fsDefault from \"node:fs\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport { Document } from \"@langchain/core/documents\";\nimport {\n UnstructuredLoaderOptions,\n UnstructuredLoader,\n} from \"../fs/unstructured.js\";\n\n/**\n * Represents the parameters for the GoogleCloudStorageLoader class. It includes\n * properties such as the GCS bucket, file destination, the options for the UnstructuredLoader and\n * the options for Google Cloud Storage\n */\nexport type GcsLoaderConfig = {\n fs?: typeof fsDefault;\n bucket: string;\n file: string;\n unstructuredLoaderOptions: UnstructuredLoaderOptions;\n storageOptions?: StorageOptions;\n};\n\n/**\n * A class that extends the BaseDocumentLoader class. It represents a\n * document loader for loading files from a google cloud storage bucket.\n * @example\n * ```typescript\n * const loader = new GoogleCloudStorageLoader({\n * bucket: \"<my-bucket-name>\",\n * file: \"<file-path>\",\n * storageOptions: {\n * keyFilename: \"<key-file-name-path>\"\n * }\n * unstructuredConfig: {\n * apiUrl: \"<unstructured-API-URL>\",\n * apiKey: \"<unstructured-API-key>\"\n * }\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class GoogleCloudStorageLoader extends BaseDocumentLoader {\n private bucket: string;\n\n private file: string;\n\n private storageOptions: StorageOptions | undefined;\n\n private _fs: typeof fsDefault;\n\n private unstructuredLoaderOptions: UnstructuredLoaderOptions;\n\n constructor({\n fs = fsDefault,\n file,\n bucket,\n unstructuredLoaderOptions,\n storageOptions,\n }: GcsLoaderConfig) {\n super();\n this._fs = fs;\n this.bucket = bucket;\n this.file = file;\n this.unstructuredLoaderOptions = unstructuredLoaderOptions;\n this.storageOptions = storageOptions;\n }\n\n async load(): Promise<Document<Record<string, unknown>>[]> {\n const tempDir = this._fs.mkdtempSync(\n path.join(os.tmpdir(), \"googlecloudstoragefileloader-\")\n );\n const filePath = path.join(tempDir, this.file);\n\n try {\n const storage = new Storage(this.storageOptions);\n const bucket = storage.bucket(this.bucket);\n\n const [buffer] = await bucket.file(this.file).download();\n this._fs.mkdirSync(path.dirname(filePath), { recursive: true });\n\n this._fs.writeFileSync(filePath, buffer);\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n } catch (e: any) {\n throw new Error(\n `Failed to download file ${this.file} from google cloud storage bucket ${this.bucket}: ${e.message}`\n );\n }\n\n try {\n const unstructuredLoader = new UnstructuredLoader(\n filePath,\n this.unstructuredLoaderOptions\n );\n const docs = await unstructuredLoader.load();\n return docs;\n } catch {\n throw new Error(\n `Failed to load file ${filePath} using unstructured loader.`\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2CA,IAAa,2BAAb,cAA8CA,0DAAmB;CAC/D,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,YAAY,EACV,KAAKC,SACL,MACA,QACA,2BACA,gBACgB,EAAE;EAClB,OAAO;EACP,KAAK,MAAM;EACX,KAAK,SAAS;EACd,KAAK,OAAO;EACZ,KAAK,4BAA4B;EACjC,KAAK,iBAAiB;CACvB;CAED,MAAM,OAAqD;EACzD,MAAM,UAAU,KAAK,IAAI,YACvBC,UAAK,KAAKC,QAAG,QAAQ,EAAE,gCAAgC,CACxD;EACD,MAAM,WAAWD,UAAK,KAAK,SAAS,KAAK,KAAK;AAE9C,MAAI;GACF,MAAM,UAAU,IAAIE,+BAAQ,KAAK;GACjC,MAAM,SAAS,QAAQ,OAAO,KAAK,OAAO;GAE1C,MAAM,CAAC,OAAO,GAAG,MAAM,OAAO,KAAK,KAAK,KAAK,CAAC,UAAU;GACxD,KAAK,IAAI,UAAUF,UAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,KAAM,EAAC;GAE/D,KAAK,IAAI,cAAc,UAAU,OAAO;EAEzC,SAAQG,GAAQ;AACf,SAAM,IAAI,MACR,CAAC,wBAAwB,EAAE,KAAK,KAAK,kCAAkC,EAAE,KAAK,OAAO,EAAE,EAAE,EAAE,SAAS;EAEvG;AAED,MAAI;GACF,MAAM,qBAAqB,IAAIC,4DAC7B,UACA,KAAK;GAEP,MAAM,OAAO,MAAM,mBAAmB,MAAM;AAC5C,UAAO;EACR,QAAO;AACN,SAAM,IAAI,MACR,CAAC,oBAAoB,EAAE,SAAS,2BAA2B,CAAC;EAE/D;CACF;AACF"}