@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 4.17 kB
Source Map (JSON)
{"version":3,"file":"google_cloud_storage.cjs","names":["BaseDocumentLoader","fsDefault","path","os","Storage","UnstructuredLoader"],"sources":["../../../src/document_loaders/web/google_cloud_storage.ts"],"sourcesContent":["import { Storage, StorageOptions } from \"@google-cloud/storage\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as fsDefault from \"node:fs\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport { Document } from \"@langchain/core/documents\";\nimport {\n UnstructuredLoaderOptions,\n UnstructuredLoader,\n} from \"../fs/unstructured.js\";\n\n/**\n * Represents the parameters for the GoogleCloudStorageLoader class. It includes\n * properties such as the GCS bucket, file destination, the options for the UnstructuredLoader and\n * the options for Google Cloud Storage\n */\nexport type GcsLoaderConfig = {\n fs?: typeof fsDefault;\n bucket: string;\n file: string;\n unstructuredLoaderOptions: UnstructuredLoaderOptions;\n storageOptions?: StorageOptions;\n};\n\n/**\n * A class that extends the BaseDocumentLoader class. It represents a\n * document loader for loading files from a google cloud storage bucket.\n * @example\n * ```typescript\n * const loader = new GoogleCloudStorageLoader({\n * bucket: \"<my-bucket-name>\",\n * file: \"<file-path>\",\n * storageOptions: {\n * keyFilename: \"<key-file-name-path>\"\n * }\n * unstructuredConfig: {\n * apiUrl: \"<unstructured-API-URL>\",\n * apiKey: \"<unstructured-API-key>\"\n * }\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class GoogleCloudStorageLoader extends BaseDocumentLoader {\n private bucket: string;\n\n private file: string;\n\n private storageOptions: StorageOptions | undefined;\n\n private _fs: typeof fsDefault;\n\n private unstructuredLoaderOptions: UnstructuredLoaderOptions;\n\n constructor({\n fs = fsDefault,\n file,\n bucket,\n unstructuredLoaderOptions,\n storageOptions,\n }: GcsLoaderConfig) {\n super();\n this._fs = fs;\n this.bucket = bucket;\n this.file = file;\n this.unstructuredLoaderOptions = unstructuredLoaderOptions;\n this.storageOptions = storageOptions;\n }\n\n async load(): Promise<Document<Record<string, unknown>>[]> {\n const tempDir = this._fs.mkdtempSync(\n path.join(os.tmpdir(), \"googlecloudstoragefileloader-\")\n );\n const filePath = path.join(tempDir, this.file);\n\n try {\n const storage = new Storage(this.storageOptions);\n const bucket = storage.bucket(this.bucket);\n\n const [buffer] = await bucket.file(this.file).download();\n this._fs.mkdirSync(path.dirname(filePath), { recursive: true });\n\n this._fs.writeFileSync(filePath, buffer);\n // oxlint-disable-next-line typescript/no-explicit-any\n } catch (e: any) {\n throw new Error(\n `Failed to download file ${this.file} from google cloud storage bucket ${this.bucket}: ${e.message}`\n );\n }\n\n try {\n const unstructuredLoader = new UnstructuredLoader(\n filePath,\n this.unstructuredLoaderOptions\n );\n const docs = await unstructuredLoader.load();\n return docs;\n } catch {\n throw new Error(\n `Failed to load file ${filePath} using unstructured loader.`\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2CA,IAAa,2BAAb,cAA8CA,sCAAAA,mBAAmB;CAC/D;CAEA;CAEA;CAEA;CAEA;CAEA,YAAY,EACV,KAAKC,SACL,MACA,QACA,2BACA,kBACkB;AAClB,SAAO;AACP,OAAK,MAAM;AACX,OAAK,SAAS;AACd,OAAK,OAAO;AACZ,OAAK,4BAA4B;AACjC,OAAK,iBAAiB;;CAGxB,MAAM,OAAqD;EACzD,MAAM,UAAU,KAAK,IAAI,YACvBC,UAAK,KAAKC,QAAG,QAAQ,EAAE,gCAAgC,CACxD;EACD,MAAM,WAAWD,UAAK,KAAK,SAAS,KAAK,KAAK;AAE9C,MAAI;GAIF,MAAM,CAAC,UAAU,MAHD,IAAIE,sBAAAA,QAAQ,KAAK,eAAe,CACzB,OAAO,KAAK,OAAO,CAEZ,KAAK,KAAK,KAAK,CAAC,UAAU;AACxD,QAAK,IAAI,UAAUF,UAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,MAAM,CAAC;AAE/D,QAAK,IAAI,cAAc,UAAU,OAAO;WAEjC,GAAQ;AACf,SAAM,IAAI,MACR,2BAA2B,KAAK,KAAK,oCAAoC,KAAK,OAAO,IAAI,EAAE,UAC5F;;AAGH,MAAI;AAMF,UADa,MAJc,IAAIG,yCAAAA,mBAC7B,UACA,KAAK,0BACN,CACqC,MAAM;UAEtC;AACN,SAAM,IAAI,MACR,uBAAuB,SAAS,6BACjC"}