@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 6.38 kB
Source Map (JSON)
{"version":3,"file":"pdf.cjs","names":["BufferLoader","filePathOrBlob: string | Blob","raw: Buffer","metadata: Document[\"metadata\"]","documents: Document[]","Document"],"sources":["../../../src/document_loaders/fs/pdf.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BufferLoader } from \"@langchain/classic/document_loaders/fs/buffer\";\n\n/**\n * A class that extends the `BufferLoader` class. It represents a document\n * loader that loads documents from PDF files.\n * @example\n * ```typescript\n * const loader = new PDFLoader(\"path/to/bitcoin.pdf\");\n * const docs = await loader.load();\n * console.log({ docs });\n * ```\n */\nexport class PDFLoader extends BufferLoader {\n private splitPages: boolean;\n\n private pdfjs: typeof PDFLoaderImports;\n\n protected parsedItemSeparator: string;\n\n constructor(\n filePathOrBlob: string | Blob,\n {\n splitPages = true,\n pdfjs = PDFLoaderImports,\n parsedItemSeparator = \"\",\n } = {}\n ) {\n super(filePathOrBlob);\n this.splitPages = splitPages;\n this.pdfjs = pdfjs;\n this.parsedItemSeparator = parsedItemSeparator;\n }\n\n /**\n * A method that takes a `raw` buffer and `metadata` as parameters and\n * returns a promise that resolves to an array of `Document` instances. It\n * uses the `getDocument` function from the PDF.js library to load the PDF\n * from the buffer. It then iterates over each page of the PDF, retrieves\n * the text content using the `getTextContent` method, and joins the text\n * items to form the page content. It creates a new `Document` instance\n * for each page with the extracted text content and metadata, and adds it\n * to the `documents` array. If `splitPages` is `true`, it returns the\n * array of `Document` instances. Otherwise, if there are no documents, it\n * returns an empty array. Otherwise, it concatenates the page content of\n * all documents and creates a single `Document` instance with the\n * concatenated content.\n * @param raw The buffer to be parsed.\n * @param metadata The metadata of the document.\n * @returns A promise that resolves to an array of `Document` instances.\n */\n public async parse(\n raw: Buffer,\n metadata: Document[\"metadata\"]\n ): Promise<Document[]> {\n const { getDocument, version } = await this.pdfjs();\n const pdf = await getDocument({\n data: new Uint8Array(raw.buffer),\n useWorkerFetch: false,\n isEvalSupported: false,\n useSystemFonts: true,\n }).promise;\n const meta = await pdf.getMetadata().catch(() => null);\n\n const documents: Document[] = [];\n\n for (let i = 1; i <= pdf.numPages; i += 1) {\n const page = await pdf.getPage(i);\n const content = await page.getTextContent();\n\n if (content.items.length === 0) {\n continue;\n }\n\n // Eliminate excessive newlines\n // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16\n let lastY;\n const textItems = [];\n for (const item of content.items) {\n if (\"str\" in item) {\n if (lastY === item.transform[5] || !lastY) {\n textItems.push(item.str);\n } else {\n textItems.push(`\\n${item.str}`);\n }\n lastY = item.transform[5];\n }\n }\n\n const text = textItems.join(this.parsedItemSeparator);\n\n documents.push(\n new Document({\n pageContent: text,\n metadata: {\n ...metadata,\n pdf: {\n version,\n info: meta?.info,\n metadata: meta?.metadata,\n totalPages: pdf.numPages,\n },\n loc: {\n pageNumber: i,\n },\n },\n })\n );\n }\n\n if (this.splitPages) {\n return documents;\n }\n\n if (documents.length === 0) {\n return [];\n }\n\n return [\n new Document({\n pageContent: documents.map((doc) => doc.pageContent).join(\"\\n\\n\"),\n metadata: {\n ...metadata,\n pdf: {\n version,\n info: meta?.info,\n metadata: meta?.metadata,\n totalPages: pdf.numPages,\n },\n },\n }),\n ];\n }\n}\n\nasync function PDFLoaderImports() {\n try {\n const { default: mod } = await import(\n \"pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js\"\n );\n const { getDocument, version } = mod;\n return { getDocument, version };\n } catch (e) {\n console.error(e);\n throw new Error(\n \"Failed to load pdf-parse. This loader currently supports pdf-parse v1 only. Please install v1, e.g. `npm install pdf-parse@^1` (v2 is not yet supported).\"\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAaA,IAAa,YAAb,cAA+BA,4DAAa;CAC1C,AAAQ;CAER,AAAQ;CAER,AAAU;CAEV,YACEC,gBACA,EACE,aAAa,MACb,QAAQ,kBACR,sBAAsB,IACvB,GAAG,CAAE,GACN;EACA,MAAM,eAAe;EACrB,KAAK,aAAa;EAClB,KAAK,QAAQ;EACb,KAAK,sBAAsB;CAC5B;;;;;;;;;;;;;;;;;;CAmBD,MAAa,MACXC,KACAC,UACqB;EACrB,MAAM,EAAE,aAAa,SAAS,GAAG,MAAM,KAAK,OAAO;EACnD,MAAM,MAAM,MAAM,YAAY;GAC5B,MAAM,IAAI,WAAW,IAAI;GACzB,gBAAgB;GAChB,iBAAiB;GACjB,gBAAgB;EACjB,EAAC,CAAC;EACH,MAAM,OAAO,MAAM,IAAI,aAAa,CAAC,MAAM,MAAM,KAAK;EAEtD,MAAMC,YAAwB,CAAE;AAEhC,OAAK,IAAI,IAAI,GAAG,KAAK,IAAI,UAAU,KAAK,GAAG;GACzC,MAAM,OAAO,MAAM,IAAI,QAAQ,EAAE;GACjC,MAAM,UAAU,MAAM,KAAK,gBAAgB;AAE3C,OAAI,QAAQ,MAAM,WAAW,EAC3B;GAKF,IAAI;GACJ,MAAM,YAAY,CAAE;AACpB,QAAK,MAAM,QAAQ,QAAQ,MACzB,KAAI,SAAS,MAAM;AACjB,QAAI,UAAU,KAAK,UAAU,MAAM,CAAC,OAClC,UAAU,KAAK,KAAK,IAAI;SAExB,UAAU,KAAK,CAAC,EAAE,EAAE,KAAK,KAAK,CAAC;IAEjC,QAAQ,KAAK,UAAU;GACxB;GAGH,MAAM,OAAO,UAAU,KAAK,KAAK,oBAAoB;GAErD,UAAU,KACR,IAAIC,oCAAS;IACX,aAAa;IACb,UAAU;KACR,GAAG;KACH,KAAK;MACH;MACA,MAAM,MAAM;MACZ,UAAU,MAAM;MAChB,YAAY,IAAI;KACjB;KACD,KAAK,EACH,YAAY,EACb;IACF;GACF,GACF;EACF;AAED,MAAI,KAAK,WACP,QAAO;AAGT,MAAI,UAAU,WAAW,EACvB,QAAO,CAAE;AAGX,SAAO,CACL,IAAIA,oCAAS;GACX,aAAa,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,KAAK,OAAO;GACjE,UAAU;IACR,GAAG;IACH,KAAK;KACH;KACA,MAAM,MAAM;KACZ,UAAU,MAAM;KAChB,YAAY,IAAI;IACjB;GACF;EACF,EACF;CACF;AACF;AAED,eAAe,mBAAmB;AAChC,KAAI;EACF,MAAM,EAAE,SAAS,KAAK,GAAG,MAAM,OAC7B;EAEF,MAAM,EAAE,aAAa,SAAS,GAAG;AACjC,SAAO;GAAE;GAAa;EAAS;CAChC,SAAQ,GAAG;EACV,QAAQ,MAAM,EAAE;AAChB,QAAM,IAAI,MACR;CAEH;AACF"}