@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 6.05 kB
Source Map (JSON)
{"version":3,"file":"s3.cjs","names":["BaseDocumentLoader","fsDefault","UnstructuredLoaderDefault","UnstructuredLoader","path","os","S3Client","GetObjectCommand","chunks: Buffer[]","Readable","chunk: Buffer","e: any"],"sources":["../../../src/document_loaders/web/s3.ts"],"sourcesContent":["import * as fsDefault from \"node:fs\";\nimport * as path from \"node:path\";\nimport * as os from \"node:os\";\nimport { Readable } from \"node:stream\";\nimport { S3Client, GetObjectCommand, S3ClientConfig } from \"@aws-sdk/client-s3\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport { UnstructuredLoader as UnstructuredLoaderDefault } from \"../fs/unstructured.js\";\n\n/**\n * Represents the parameters for the S3Loader class. It includes\n * properties such as the S3 bucket, key, unstructured API URL,\n * unstructured API key, S3 configuration, file system module, and\n * UnstructuredLoader module.\n */\nexport interface S3LoaderParams {\n bucket: string;\n key: string;\n unstructuredAPIURL: string;\n unstructuredAPIKey: string;\n s3Config?: S3ClientConfig;\n fs?: typeof fsDefault;\n UnstructuredLoader?: typeof UnstructuredLoaderDefault;\n}\n\n/**\n * A class that extends the BaseDocumentLoader class. It represents a\n * document loader for loading files from an S3 bucket.\n * @example\n * ```typescript\n * const loader = new S3Loader({\n * bucket: \"my-document-bucket-123\",\n * key: \"AccountingOverview.pdf\",\n * s3Config: {\n * region: \"us-east-1\",\n * credentials: {\n * accessKeyId: \"<YourAccessKeyId>\",\n * secretAccessKey: \"<YourSecretAccessKey>\",\n * },\n * },\n * unstructuredAPIURL: \"<YourUnstructuredAPIURL>\",\n * unstructuredAPIKey: \"<YourUnstructuredAPIKey>\",\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class S3Loader extends BaseDocumentLoader {\n private bucket: string;\n\n private key: string;\n\n private unstructuredAPIURL: string;\n\n private unstructuredAPIKey: string;\n\n private s3Config: S3ClientConfig;\n\n private _fs: typeof fsDefault;\n\n private _UnstructuredLoader: typeof UnstructuredLoaderDefault;\n\n constructor({\n bucket,\n key,\n unstructuredAPIURL,\n unstructuredAPIKey,\n s3Config = {},\n fs = fsDefault,\n UnstructuredLoader = UnstructuredLoaderDefault,\n }: S3LoaderParams) {\n super();\n this.bucket = bucket;\n this.key = key;\n this.unstructuredAPIURL = unstructuredAPIURL;\n this.unstructuredAPIKey = unstructuredAPIKey;\n this.s3Config = s3Config;\n this._fs = fs;\n this._UnstructuredLoader = UnstructuredLoader;\n }\n\n /**\n * Loads the file from the S3 bucket, saves it to a temporary directory,\n * and then uses the UnstructuredLoader to load the file as a document.\n * @returns An array of Document objects representing the loaded documents.\n */\n public async load() {\n const tempDir = this._fs.mkdtempSync(\n path.join(os.tmpdir(), \"s3fileloader-\")\n );\n\n const filePath = path.join(tempDir, this.key);\n\n try {\n const s3Client = new S3Client(this.s3Config);\n\n const getObjectCommand = new GetObjectCommand({\n Bucket: this.bucket,\n Key: this.key,\n });\n\n const response = await s3Client.send(getObjectCommand);\n\n const objectData = await new Promise<Buffer>((resolve, reject) => {\n const chunks: Buffer[] = [];\n\n // eslint-disable-next-line no-instanceof/no-instanceof\n if (response.Body instanceof Readable) {\n response.Body.on(\"data\", (chunk: Buffer) => chunks.push(chunk));\n response.Body.on(\"end\", () => resolve(Buffer.concat(chunks)));\n response.Body.on(\"error\", reject);\n } else {\n reject(new Error(\"Response body is not a readable stream.\"));\n }\n });\n\n this._fs.mkdirSync(path.dirname(filePath), { recursive: true });\n\n this._fs.writeFileSync(filePath, objectData);\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n } catch (e: any) {\n throw new Error(\n `Failed to download file ${this.key} from S3 bucket ${this.bucket}: ${e.message}`\n );\n }\n\n try {\n const options = {\n apiUrl: this.unstructuredAPIURL,\n apiKey: this.unstructuredAPIKey,\n };\n\n const unstructuredLoader = new this._UnstructuredLoader(\n filePath,\n options\n );\n\n const docs = await unstructuredLoader.load();\n\n return docs;\n } catch {\n throw new Error(\n `Failed to load file ${filePath} using unstructured loader.`\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CA,IAAa,WAAb,cAA8BA,0DAAmB;CAC/C,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,YAAY,EACV,QACA,KACA,oBACA,oBACA,WAAW,CAAE,GACb,KAAKC,SACL,2CAAqBC,6DACN,EAAE;EACjB,OAAO;EACP,KAAK,SAAS;EACd,KAAK,MAAM;EACX,KAAK,qBAAqB;EAC1B,KAAK,qBAAqB;EAC1B,KAAK,WAAW;EAChB,KAAK,MAAM;EACX,KAAK,sBAAsBC;CAC5B;;;;;;CAOD,MAAa,OAAO;EAClB,MAAM,UAAU,KAAK,IAAI,YACvBC,UAAK,KAAKC,QAAG,QAAQ,EAAE,gBAAgB,CACxC;EAED,MAAM,WAAWD,UAAK,KAAK,SAAS,KAAK,IAAI;AAE7C,MAAI;GACF,MAAM,WAAW,IAAIE,6BAAS,KAAK;GAEnC,MAAM,mBAAmB,IAAIC,qCAAiB;IAC5C,QAAQ,KAAK;IACb,KAAK,KAAK;GACX;GAED,MAAM,WAAW,MAAM,SAAS,KAAK,iBAAiB;GAEtD,MAAM,aAAa,MAAM,IAAI,QAAgB,CAAC,SAAS,WAAW;IAChE,MAAMC,SAAmB,CAAE;AAG3B,QAAI,SAAS,gBAAgBC,sBAAU;KACrC,SAAS,KAAK,GAAG,QAAQ,CAACC,UAAkB,OAAO,KAAK,MAAM,CAAC;KAC/D,SAAS,KAAK,GAAG,OAAO,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC;KAC7D,SAAS,KAAK,GAAG,SAAS,OAAO;IAClC,OACC,uBAAO,IAAI,MAAM,2CAA2C;GAE/D;GAED,KAAK,IAAI,UAAUN,UAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,KAAM,EAAC;GAE/D,KAAK,IAAI,cAAc,UAAU,WAAW;EAE7C,SAAQO,GAAQ;AACf,SAAM,IAAI,MACR,CAAC,wBAAwB,EAAE,KAAK,IAAI,gBAAgB,EAAE,KAAK,OAAO,EAAE,EAAE,EAAE,SAAS;EAEpF;AAED,MAAI;GACF,MAAM,UAAU;IACd,QAAQ,KAAK;IACb,QAAQ,KAAK;GACd;GAED,MAAM,qBAAqB,IAAI,KAAK,oBAClC,UACA;GAGF,MAAM,OAAO,MAAM,mBAAmB,MAAM;AAE5C,UAAO;EACR,QAAO;AACN,SAAM,IAAI,MACR,CAAC,oBAAoB,EAAE,SAAS,2BAA2B,CAAC;EAE/D;CACF;AACF"}